Clone and install requirements for CPPlugins (#91)

* Clone the plugins repo

* Update Dockerfile

* Add plugins directory to CP call

* Add plugins to config

* Add USE_PLUGINS

* Fix -i bug in batchfiles, make plugins follow an input variable

* Fix formatting of cmd lines

* Space before --plugins

* Rebase plugins use (#109)

* Add userdata for each LaunchSpecification (#102)

Co-authored-by: Ubuntu <[email protected]>

* Batch files should be able to have input files too

Co-authored-by: Matthew Chess <[email protected]>
Co-authored-by: Ubuntu <[email protected]>

Co-authored-by: Matthew Chess <[email protected]>
Co-authored-by: Ubuntu <[email protected]>

Author: 3 people

config.py

@@ -37,3 +37,6 @@
 CHECK_IF_DONE_BOOL = 'True'  #True or False- should it check if there are a certain number of non-empty files and delete the job if yes?
 EXPECTED_NUMBER_FILES = 7    #What is the number of files that trigger skipping a job?
 MIN_FILE_SIZE_BYTES = 1      #What is the minimal number of bytes an object should be to "count"?
+
+# PLUGINS
+USE_PLUGINS = 'True'

fabfile.py

@@ -158,6 +158,10 @@ def generate_task_definition():
 	{
 	    "name": "MIN_FILE_SIZE_BYTES",
 	    "value": str(MIN_FILE_SIZE_BYTES)
+	},
+	{
+	    "name": "USE_PLUGINS",
+	    "value": str(USE_PLUGINS)
 	}
     ]
     return task_definition

worker/Dockerfile

@@ -5,6 +5,7 @@
 # This wraps the cellprofiler docker registry
 #
 
+
 FROM cellprofiler/cellprofiler:3.1.9
 
 # Install S3FS 
@@ -56,6 +57,12 @@ COPY instance-monitor.py .
 COPY run-worker.sh .
 RUN chmod 755 run-worker.sh
 
+RUN git clone https://github.com/CellProfiler/CellProfiler-plugins.git
+WORKDIR /home/ubuntu/CellProfiler-plugins
+RUN \
+    pip install -r requirements.txt
+
+WORKDIR /home/ubuntu
 ENTRYPOINT ["./run-worker.sh"]
 CMD [""]
 

worker/cp-worker.py

@@ -17,6 +17,7 @@
 
 DATA_ROOT = '/home/ubuntu/bucket'
 LOCAL_OUTPUT = '/home/ubuntu/local_output'
+PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins'
 QUEUE_URL = os.environ['SQS_QUEUE_URL']
 AWS_BUCKET = os.environ['AWS_BUCKET']
 LOG_GROUP_NAME= os.environ['LOG_GROUP_NAME']
@@ -26,6 +27,10 @@
     MIN_FILE_SIZE_BYTES = 1
 else:
     MIN_FILE_SIZE_BYTES = int(os.environ['MIN_FILE_SIZE_BYTES'])
+if 'USE_PLUGINS' not in os.environ:
+    USE_PLUGINS = 'False'
+else:
+    USE_PLUGINS = os.environ['USE_PLUGINS']
 
 #################################
 # CLASS TO HANDLE THE SQS QUEUE
@@ -129,7 +134,7 @@ def runCellProfiler(message):
     remoteOut= os.path.join(message['output'],metadataID)
     replaceValues = {'PL':message['pipeline'], 'OUT':localOut, 'FL':message['data_file'],
 			'DATA': DATA_ROOT, 'Metadata': message['Metadata'], 'IN': message['input'], 
-			'MetadataID':metadataID }
+			'MetadataID':metadataID, 'PLUGINS':PLUGIN_DIR }
     # See if this is a message you've already handled, if you've so chosen
     if CHECK_IF_DONE_BOOL.upper() == 'TRUE':
         try:
@@ -159,9 +164,12 @@ def runCellProfiler(message):
     cpDone = localOut + '/cp.is.done'
     if message['pipeline'][-3:]!='.h5':
         cmd = cmdstem + '-p %(DATA)s/%(PL)s -i %(DATA)s/%(IN)s -o %(OUT)s -d ' + cpDone
-        cmd += ' --data-file=%(DATA)s/%(FL)s -g %(Metadata)s'
+        cmd += ' --data-file=%(DATA)s/%(FL)s '
+        cmd += '-g %(Metadata)s'
     else:
         cmd = cmdstem + '-p %(DATA)s/%(PL)s -i %(DATA)s/%(IN)s -o %(OUT)s -d ' + cpDone + ' --data-file=%(DATA)s/%(FL)s -g %(Metadata)s'
+    if USE_PLUGINS == 'True':
+        cmd += ' --plugins-directory=%(PLUGINS)s'
     cmd = cmd % replaceValues
     print('Running', cmd)
     logger.info(cmd)