Homec4science
Diffusion tfbs_annotate_FIMO (master) Browse

R10523/scripts/run_fimo.py
73d080087c1cmaster

Last Change
Raw File

run_fimo.py
View Options

#! /usr/bin/env python3
import os
import sys
import subprocess
import shutil
import multiprocessing
import shlex
import pandas as pd
"""argv[1]: path to root of the tfbs_annotate_fimo project"""
project_root_dir = sys.argv[1]
# get username on SCITAS server
process = subprocess.run('echo $USER', shell=True,
stdout=subprocess.PIPE)
username = process.stdout.decode().rstrip()
scratch_path = '/scratch/'+username
split_fasta_path = os.path.join(scratch_path, 'split_fasta')
(_, _, fasta_files) = next(os.walk(split_fasta_path))
fasta_files.sort()
ncores = len(fasta_files)
# multiprocessing code from https://stackoverflow.com/questions/25120363/multiprocessing-execute-external-command-and-wait-before-proceeding
from multiprocessing.pool import ThreadPool
def call_proc(cmd):
""" This runs in a separate thread. """
p = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
return(out, err)
pool = ThreadPool(ncores)
results = []
fimo_path = os.path.join(scratch_path, 'fimo')
if os.path.exists(fimo_path):
shutil.rmtree(fimo_path)
os.mkdir(fimo_path)
pwms_path = os.path.join(project_root_dir, 'data', 'pwms_cisbp_meme.txt')
for i, f in enumerate(fasta_files):
# creating a subdirectory per fimo process to avoid write conflicts
fimo_out_path = os.path.join(fimo_path, 'fimo_out_'+str(i))
os.mkdir(fimo_out_path)
f_path = os.path.join(split_fasta_path, f)
command = "fimo --no-qvalue --oc {} {} {}".format(fimo_out_path, pwms_path, f_path)
print(command)
results.append(pool.apply_async(call_proc, (command,)))
# Close the pool and wait for each running task to complete
pool.close()
pool.join()
# printing logs
for r in results:
out, err = r.get()
with open(os.path.join(fimo_path, 'fimo_log.txt'), 'w') as f:
print("out: {} err: {}\n".format(out.decode(), err.decode()), file=f)
# reassembling fimo results in a single file
(_, dirs, _) = next(os.walk(fimo_path))
fimo_res_concat = pd.read_csv(os.path.join(fimo_path, dirs[0], 'fimo.tsv'),
sep='\t', header=0, index_col=False)
for d in dirs[1:]:
fimo_res_temp = pd.read_csv(os.path.join(fimo_path, d, 'fimo.tsv'),
sep='\t', header=0, index_col=False)
fimo_res_concat = fimo_res_concat.append(fimo_res_temp, ignore_index=True)
# saving fimo result table
results_path = os.path.join(project_root_dir, 'results', 'fimo_res_concat.tsv')
if os.path.exists(results_path):
os.remove(results_path)
fimo_res_concat.to_csv(results_path, sep='\t', header=True, index=False)
c4science · Help