diff --git a/.gitignore b/.gitignore deleted file mode 100644 index b6e4761..0000000 --- a/.gitignore +++ /dev/null @@ -1,129 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f19f79e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# Code for AWS INSTALL +## download image +#FROM public.ecr.aws/lambda/python:3.6 +# +## Copy function code +#COPY app/lambda_predict.py ./ +#COPY requirements.txt ./ +#COPY setup.py ./ +# +#RUN pip install --upgrade pip && pip install -r requirements.txt +# +#COPY automorph/ ./automorph +#RUN pip install automorph/ +# +#ENV TORCH_HOME=/tmp +# +#CMD ["lambda_predict.lambda_handler"] + +#AZURE attempt +#FROM 10.0.0.95:5000/arajesh/automorph_lee +FROM python:3.6 + +# Copy function code +COPY requirements.txt ./ +COPY setup.py ./ +# +RUN pip install --upgrade pip && pip install -r requirements.txt +# +##cv2 dependencies +RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y + +COPY automorph/ ./automorph +RUN python setup.py install + diff --git a/M0_Preprocess/EyeQ_process_main.py b/M0_Preprocess/EyeQ_process_main.py deleted file mode 100644 index 012f4fe..0000000 --- a/M0_Preprocess/EyeQ_process_main.py +++ /dev/null @@ -1,68 +0,0 @@ -import fundus_prep as prep -import glob -import os -import cv2 as cv -import pandas as pd -from PIL import ImageFile -import shutil -ImageFile.LOAD_TRUNCATED_IMAGES = True - -def process(image_list, save_path): - - radius_list = [] - centre_list_w = [] - centre_list_h = [] - name_list = [] - list_resolution = [] - scale_resolution = [] - - resolution_list = pd.read_csv('../resolution_information.csv') - - for image_path in image_list: - - dst_image = '../images/' + image_path - if os.path.exists('../Results/M0/images/' + image_path): - print('continue...') - continue - - try: - resolution_ = resolution_list['res'][resolution_list['fundus']==image_path].values[0] - list_resolution.append(resolution_) - - img = prep.imread(dst_image) - r_img, borders, mask, r_img, radius_list,centre_list_w, centre_list_h = prep.process_without_gb(img,img,radius_list,centre_list_w, centre_list_h) - prep.imwrite(save_path + image_path.split('.')[0] + '.png', r_img) - #prep.imwrite('../Results/M0/images/' + image_path.split('.')[0] + '.png', mask) - - - name_list.append(image_path.split('.')[0] + '.png') - - except: - pass - - scale_list = [a*2/912 for a in radius_list] - - scale_resolution = [a*b*1000 for a,b in zip(list_resolution,scale_list)] - - Data4stage2 = pd.DataFrame({'Name':name_list, 'centre_w':centre_list_w, 'centre_h':centre_list_h, 'radius':radius_list, 'Scale':scale_list, 'Scale_resolution':scale_resolution}) - Data4stage2.to_csv('../Results/M0/crop_info.csv', index = None, encoding='utf8') - - - - - -if __name__ == "__main__": - if os.path.exists('../images/.ipynb_checkpoints'): - shutil.rmtree('../images/.ipynb_checkpoints') - image_list = sorted(os.listdir('../images')) - save_path = '../Results/M0/images/' - if not os.path.exists(save_path): - os.makedirs(save_path) - - process(image_list, save_path) - - - - - - diff --git a/M1_Retinal_Image_quality_EyePACS/merge_quality_assessment.py b/M1_Retinal_Image_quality_EyePACS/merge_quality_assessment.py deleted file mode 100644 index 66c6b64..0000000 --- a/M1_Retinal_Image_quality_EyePACS/merge_quality_assessment.py +++ /dev/null @@ -1,40 +0,0 @@ -import numpy as np -import pandas as pd -import shutil -import os - - -result_Eyepacs = './test_outside/results_ensemble.csv' - -if not os.path.exists('../Results/M1/Good_quality/'): - os.makedirs('../Results/M1/Good_quality/') -if not os.path.exists('../Results/M1/Bad_quality/'): - os.makedirs('../Results/M1/Bad_quality/') - -result_Eyepacs_ = pd.read_csv(result_Eyepacs) - -Eyepacs_pre = result_Eyepacs_['Prediction'] -Eyepacs_bad_mean = result_Eyepacs_['softmax_bad'] -Eyepacs_usable_sd = result_Eyepacs_['usable_sd'] -name_list = result_Eyepacs_['Name'] - -Eye_good = 0 -Eye_bad = 0 - -for i in range(len(name_list)): - - if Eyepacs_pre[i]==0: - Eye_good+=1 - shutil.copy(name_list[i], '../Results/M1/Good_quality/') - elif (Eyepacs_pre[i]==1) and (Eyepacs_bad_mean[i]<0.25): - #elif (Eyepacs_pre[i]==1) and (Eyepacs_bad_mean[i]<0.25) and (Eyepacs_usable_sd[i]<0.1): - Eye_good+=1 - shutil.copy(name_list[i], '../Results/M1/Good_quality/') - else: - Eye_bad+=1 - shutil.copy(name_list[i], '../Results/M1/Bad_quality/') - #shutil.copy(name_list[i], '../Results/M1/Good_quality/') - - -print('Gradable cases by EyePACS_QA is {} '.format(Eye_good)) -print('Ungradable cases by EyePACS_QA is {} '.format(Eye_bad)) diff --git a/M1_Retinal_Image_quality_EyePACS/test_outside.sh b/M1_Retinal_Image_quality_EyePACS/test_outside.sh deleted file mode 100644 index 639885e..0000000 --- a/M1_Retinal_Image_quality_EyePACS/test_outside.sh +++ /dev/null @@ -1,20 +0,0 @@ - -CUDA_NUMBER=0 - -export PYTHONPATH=.:$PYTHONPATH - -for model in 'efficientnet' -#for model in 'densenet169' -do - for n_round in 0 - do - seed_number=$((42-2*n_round)) - CUDA_VISIBLE_DEVICES=${CUDA_NUMBER} python test_outside.py --e=1 --b=64 --task_name='Retinal_quality' --model=${model} --round=${n_round} --train_on_dataset='EyePACS_quality' \ - --test_on_dataset='customised_data' --test_csv_dir='../Results/M0/images/' --n_class=3 --seed_num=${seed_number} - - - done - -done - - diff --git a/M1_Retinal_Image_quality_EyePACS/test_outside/results_ensemble.csv b/M1_Retinal_Image_quality_EyePACS/test_outside/results_ensemble.csv deleted file mode 100644 index c4676da..0000000 --- a/M1_Retinal_Image_quality_EyePACS/test_outside/results_ensemble.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,softmax_good,softmax_usable,softmax_bad,good_sd,usable_sd,bad_sd,Prediction -../Results/M0/images/1.png,0.22234638,0.38861126,0.38904238,0.22372797,0.13229463,0.2042717,2 -../Results/M0/images/100.png,0.40094352,0.43912467,0.15993184,0.2513004,0.16993268,0.16648127,1 -../Results/M0/images/11.png,0.91802436,0.07071902,0.011256653,0.11915948,0.09239171,0.027682042,0 diff --git a/M3_feature_whole_pic/retipy/create_datasets_macular_centred.py b/M3_feature_whole_pic/retipy/create_datasets_macular_centred.py deleted file mode 100644 index 9908947..0000000 --- a/M3_feature_whole_pic/retipy/create_datasets_macular_centred.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 - -# Retipy - Retinal Image Processing on Python -# Copyright (C) 2017 Alejandro Valdes -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -script to estimate the linear tortuosity of a set of retinal images, it will output the values -to a file in the output folder defined in the configuration. The output will only have the -estimated value and it is sorted by image file name. -""" - -import argparse -import glob -# import numpy as np -import os -import h5py -import shutil -import pandas as pd -# import scipy.stats as stats - -from retipy import configuration, retina, tortuosity_measures - - -if os.path.exists('../../Results/M2/artery_vein/artery_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/artery_binary_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/binary_vessel/binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/binary_vessel/binary_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/artery_vein/vein_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/vein_binary_skeleton/.ipynb_checkpoints') -if not os.path.exists('../../Results/M3/Macular_centred/Width/'): - os.makedirs('../../Results/M3/Macular_centred/Width/') - -#if os.path.exists('./DDR/av_seg/raw/.ipynb_checkpoints'): -# shutil.rmtree('./DDR/av_seg/raw/.ipynb_checkpoints') - - -parser = argparse.ArgumentParser() - -parser.add_argument( - "-c", - "--configuration", - help="the configuration file location", - default="resources/retipy.config") -args = parser.parse_args() - -CONFIG = configuration.Configuration(args.configuration) -binary_FD_binary,binary_VD_binary,binary_Average_width,binary_t2_list,binary_t4_list,binary_t5_list = [],[],[],[],[],[] -artery_FD_binary,artery_VD_binary,artery_Average_width,artery_t2_list,artery_t4_list,artery_t5_list = [],[],[],[],[],[] -vein_FD_binary,vein_VD_binary,vein_Average_width,vein_t2_list,vein_t4_list,vein_t5_list = [],[],[],[],[],[] -name_list = [] - -Artery_PATH = '../../Results/M2/artery_vein/macular_centred_artery_skeleton' -Vein_PATH = '../../Results/M2/artery_vein/macular_centred_vein_skeleton' -Binary_PATH = '../../Results/M2/binary_vessel/macular_centred_binary_skeleton' - -for filename in sorted(glob.glob(os.path.join(Binary_PATH, '*.png'))): - segmentedImage = retina.Retina(None, filename, store_path='../../Results/M2/binary_vessel/macular_centred_binary_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width, t2, t4, td = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/binary_vessel/macular_centred_binary_process/') - #print(window.tags) - binary_t2_list.append(t2) - binary_t4_list.append(t4) - binary_t5_list.append(td) - binary_FD_binary.append(FD_binary) - binary_VD_binary.append(VD_binary) - binary_Average_width.append(Average_width) - name_list.append(filename.split('/')[-1]) - - -for filename in sorted(glob.glob(os.path.join(Artery_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/macular_centred_artery_process') - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width, t2, t4, td = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/macular_centred_artery_process/') - #print(window.tags) - artery_t2_list.append(t2) - artery_t4_list.append(t4) - artery_t5_list.append(td) - artery_FD_binary.append(FD_binary) - artery_VD_binary.append(VD_binary) - artery_Average_width.append(Average_width) - - -for filename in sorted(glob.glob(os.path.join(Vein_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/macular_centred_vein_process') - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width, t2, t4, td = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/macular_centred_vein_process/') - #print(window.tags) - vein_t2_list.append(t2) - vein_t4_list.append(t4) - vein_t5_list.append(td) - vein_FD_binary.append(FD_binary) - vein_VD_binary.append(VD_binary) - vein_Average_width.append(Average_width) - -Disc_file = pd.read_csv('../../Results/M3/Macular_centred/Disc_cup_results.csv') - -Data4stage2 = pd.DataFrame({'Fractal_dimension':binary_FD_binary, 'Vessel_density':binary_VD_binary, 'Average_width':binary_Average_width,'Distance_tortuosity':binary_t2_list, 'Squared_curvature_tortuosity':binary_t4_list, 'Tortuosity_density':binary_t5_list, 'Artery_Fractal_dimension':artery_FD_binary, 'Artery_Vessel_density':artery_VD_binary, 'Artery_Average_width':artery_Average_width,'Artery_Distance_tortuosity':artery_t2_list, 'Artery_Squared_curvature_tortuosity':artery_t4_list, 'Artery_Tortuosity_density':artery_t5_list, 'Vein_Fractal_dimension':vein_FD_binary, 'Vein_Vessel_density':vein_VD_binary, 'Vein_Average_width':vein_Average_width,'Vein_Distance_tortuosity':vein_t2_list, 'Vein_Squared_curvature_tortuosity':vein_t4_list, 'Vein_Tortuosity_density':vein_t5_list}) -#Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index().drop(columns=['index', 'Name']), Data4stage2] ,axis=1) -Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index(), Data4stage2] ,axis=1) -Data4stage2.to_csv('../../Results/M3/Macular_centred/Macular_Measurement.csv', index = None, encoding='utf8') diff --git a/M3_feature_zone/retipy/create_datasets_disc_centred_B.py b/M3_feature_zone/retipy/create_datasets_disc_centred_B.py deleted file mode 100644 index 42d412e..0000000 --- a/M3_feature_zone/retipy/create_datasets_disc_centred_B.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 - -# Retipy - Retinal Image Processing on Python -# Copyright (C) 2017 Alejandro Valdes -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -script to estimate the linear tortuosity of a set of retinal images, it will output the values -to a file in the output folder defined in the configuration. The output will only have the -estimated value and it is sorted by image file name. -""" - -import argparse -import glob -# import numpy as np -import os -import h5py -import shutil -import pandas as pd -# import scipy.stats as stats - -from retipy import configuration, retina, tortuosity_measures - - -if os.path.exists('../../Results/M2/artery_vein/Zone_B_disc_centred_artery_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/Zone_B_disc_centred_artery_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/artery_vein/Zone_B_disc_centred_vein_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/Zone_B_disc_centred_vein_skeleton/.ipynb_checkpoints') -if not os.path.exists('../../Results/M3/Disc_centred/Width/'): - os.makedirs('../../Results/M3/Disc_centred/Width/') - -#if os.path.exists('./DDR/av_seg/raw/.ipynb_checkpoints'): -# shutil.rmtree('./DDR/av_seg/raw/.ipynb_checkpoints') - - -parser = argparse.ArgumentParser() - -parser.add_argument( - "-c", - "--configuration", - help="the configuration file location", - default="resources/retipy.config") -args = parser.parse_args() - - -CONFIG = configuration.Configuration(args.configuration) -binary_FD_binary,binary_VD_binary,binary_Average_width,binary_t2_list,binary_t4_list,binary_t5_list = [],[],[],[],[],[] -artery_FD_binary,artery_VD_binary,artery_Average_width,artery_t2_list,artery_t4_list,artery_t5_list = [],[],[],[],[],[] -vein_FD_binary,vein_VD_binary,vein_Average_width,vein_t2_list,vein_t4_list,vein_t5_list = [],[],[],[],[],[] -CRAE_Hubbard_list = [] -CRVE_Hubbard_list = [] -AVR_Hubbard_list = [] -CRAE_Knudtson_list = [] -CRVE_Knudtson_list = [] -AVR_Knudtson_list = [] -name_list = [] - -Artery_PATH = '../../Results/M2/artery_vein/Zone_B_disc_centred_artery_skeleton' -Vein_PATH = '../../Results/M2/artery_vein/Zone_B_disc_centred_vein_skeleton' -Binary_PATH = '../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_skeleton' - -for filename in sorted(glob.glob(os.path.join(Binary_PATH, '*.png'))): - segmentedImage = retina.Retina(None, filename, store_path='../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average, _, _,_,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_process/') - #print(window.tags) - binary_t2_list.append(t2) - binary_t4_list.append(t4) - binary_t5_list.append(td) - binary_FD_binary.append(FD_binary) - binary_VD_binary.append(VD_binary) - binary_Average_width.append(Average_width) - name_list.append(filename.split('/')[-1]) - - - - - -for filename in sorted(glob.glob(os.path.join(Artery_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_artery_process') - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,CRAE_Hubbard, _,CRAE_Knudtson,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_artery_process/') - #print(window.tags) - artery_t2_list.append(t2) - artery_t4_list.append(t4) - artery_t5_list.append(td) - artery_FD_binary.append(FD_binary) - artery_VD_binary.append(VD_binary) - artery_Average_width.append(Average_width) - CRAE_Hubbard_list.append(CRAE_Hubbard) - CRAE_Knudtson_list.append(CRAE_Knudtson) - - - -####################################3 - - -for filename in sorted(glob.glob(os.path.join(Vein_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_vein_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,_, CRVE_Hubbard,_,CRVE_Knudtson = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_vein_process/') - #print(window.tags) - vein_t2_list.append(t2) - vein_t4_list.append(t4) - vein_t5_list.append(td) - vein_FD_binary.append(FD_binary) - vein_VD_binary.append(VD_binary) - vein_Average_width.append(Average_width) - CRVE_Hubbard_list.append(CRVE_Hubbard) - CRVE_Knudtson_list.append(CRVE_Knudtson) - - -AVR_Knudtson_list = [a / b for a,b in zip(CRAE_Knudtson_list, CRVE_Knudtson_list)] -AVR_Hubbard_list = [a / b for a,b in zip(CRAE_Hubbard_list, CRVE_Hubbard_list)] - -Disc_file = pd.read_csv('../../Results/M3/Disc_centred/Disc_cup_results.csv') - -Data4stage2 = pd.DataFrame({'Fractal_dimension':binary_FD_binary, 'Vessel_density':binary_VD_binary, 'Average_width':binary_Average_width,'Distance_tortuosity':binary_t2_list, 'Squared_curvature_tortuosity':binary_t4_list, 'Tortuosity_density':binary_t5_list, 'Artery_Fractal_dimension':artery_FD_binary, 'Artery_Vessel_density':artery_VD_binary, 'Artery_Average_width':artery_Average_width,'Artery_Distance_tortuosity':artery_t2_list, 'Artery_Squared_curvature_tortuosity':artery_t4_list, 'Artery_Tortuosity_density':artery_t5_list, 'Vein_Fractal_dimension':vein_FD_binary, 'Vein_Vessel_density':vein_VD_binary, 'Vein_Average_width':vein_Average_width,'Vein_Distance_tortuosity':vein_t2_list, 'Vein_Squared_curvature_tortuosity':vein_t4_list, 'Vein_Tortuosity_density':vein_t5_list, 'CRAE_Hubbard':CRAE_Hubbard_list, 'CRVE_Hubbard':CRVE_Hubbard_list, 'AVR_Hubbard':AVR_Hubbard_list, 'CRAE_Knudtson':CRAE_Knudtson_list, 'CRVE_Knudtson':CRVE_Knudtson_list, 'AVR_Knudtson':AVR_Knudtson_list}) -#Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index().drop(columns=['index', 'Name']), Data4stage2] ,axis=1) -Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index(), Data4stage2] ,axis=1) -Data4stage2.to_csv('../../Results/M3/Disc_centred/Disc_Zone_B_Measurement.csv', index = None, encoding='utf8') - - diff --git a/M3_feature_zone/retipy/create_datasets_disc_centred_C.py b/M3_feature_zone/retipy/create_datasets_disc_centred_C.py deleted file mode 100644 index b51b5d9..0000000 --- a/M3_feature_zone/retipy/create_datasets_disc_centred_C.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 - -# Retipy - Retinal Image Processing on Python -# Copyright (C) 2017 Alejandro Valdes -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -script to estimate the linear tortuosity of a set of retinal images, it will output the values -to a file in the output folder defined in the configuration. The output will only have the -estimated value and it is sorted by image file name. -""" - -import argparse -import glob -# import numpy as np -import os -import h5py -import shutil -import pandas as pd -# import scipy.stats as stats - -from retipy import configuration, retina, tortuosity_measures - - -if os.path.exists('../../Results/M2/artery_vein/Zone_B_disc_centred_artery_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/Zone_B_disc_centred_artery_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/artery_vein/Zone_B_disc_centred_vein_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/Zone_B_disc_centred_vein_skeleton/.ipynb_checkpoints') -if not os.path.exists('../../Results/M3/Disc_centred/Width/'): - os.makedirs('../../Results/M3/Disc_centred/Width/') - -#if os.path.exists('./DDR/av_seg/raw/.ipynb_checkpoints'): -# shutil.rmtree('./DDR/av_seg/raw/.ipynb_checkpoints') - - -parser = argparse.ArgumentParser() - -parser.add_argument( - "-c", - "--configuration", - help="the configuration file location", - default="resources/retipy.config") -args = parser.parse_args() - - -CONFIG = configuration.Configuration(args.configuration) -binary_FD_binary,binary_VD_binary,binary_Average_width,binary_t2_list,binary_t4_list,binary_t5_list = [],[],[],[],[],[] -artery_FD_binary,artery_VD_binary,artery_Average_width,artery_t2_list,artery_t4_list,artery_t5_list = [],[],[],[],[],[] -vein_FD_binary,vein_VD_binary,vein_Average_width,vein_t2_list,vein_t4_list,vein_t5_list = [],[],[],[],[],[] -CRAE_Hubbard_list = [] -CRVE_Hubbard_list = [] -AVR_Hubbard_list = [] -CRAE_Knudtson_list = [] -CRVE_Knudtson_list = [] -AVR_Knudtson_list = [] -name_list = [] - -Artery_PATH = '../../Results/M2/artery_vein/Zone_B_disc_centred_artery_skeleton' -Vein_PATH = '../../Results/M2/artery_vein/Zone_B_disc_centred_vein_skeleton' -Binary_PATH = '../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_skeleton' - -for filename in sorted(glob.glob(os.path.join(Binary_PATH, '*.png'))): - segmentedImage = retina.Retina(None, filename, store_path='../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average, _, _,_,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/binary_vessel/Zone_B_disc_centred_binary_process/') - #print(window.tags) - binary_t2_list.append(t2) - binary_t4_list.append(t4) - binary_t5_list.append(td) - binary_FD_binary.append(FD_binary) - binary_VD_binary.append(VD_binary) - binary_Average_width.append(Average_width) - name_list.append(filename.split('/')[-1]) - - - - - -for filename in sorted(glob.glob(os.path.join(Artery_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_artery_process') - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,CRAE_Hubbard, _,CRAE_Knudtson,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_artery_process/') - #print(window.tags) - artery_t2_list.append(t2) - artery_t4_list.append(t4) - artery_t5_list.append(td) - artery_FD_binary.append(FD_binary) - artery_VD_binary.append(VD_binary) - artery_Average_width.append(Average_width) - CRAE_Hubbard_list.append(CRAE_Hubbard) - CRAE_Knudtson_list.append(CRAE_Knudtson) - - - - -####################################3 - - -for filename in sorted(glob.glob(os.path.join(Vein_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_vein_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,_, CRVE_Hubbard,_,CRVE_Knudtson = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/Zone_B_disc_centred_vein_process/') - #print(window.tags) - vein_t2_list.append(t2) - vein_t4_list.append(t4) - vein_t5_list.append(td) - vein_FD_binary.append(FD_binary) - vein_VD_binary.append(VD_binary) - vein_Average_width.append(Average_width) - CRVE_Hubbard_list.append(CRVE_Hubbard) - CRVE_Knudtson_list.append(CRVE_Knudtson) - - - -AVR_Knudtson_list = [a / b for a,b in zip(CRAE_Knudtson_list, CRVE_Knudtson_list)] -AVR_Hubbard_list = [a / b for a,b in zip(CRAE_Hubbard_list, CRVE_Hubbard_list)] - -Disc_file = pd.read_csv('../../Results/M3/Disc_centred/Disc_cup_results.csv') - -Data4stage2 = pd.DataFrame({'Fractal_dimension':binary_FD_binary, 'Vessel_density':binary_VD_binary, 'Average_width':binary_Average_width,'Distance_tortuosity':binary_t2_list, 'Squared_curvature_tortuosity':binary_t4_list, 'Tortuosity_density':binary_t5_list, 'Artery_Fractal_dimension':artery_FD_binary, 'Artery_Vessel_density':artery_VD_binary, 'Artery_Average_width':artery_Average_width,'Artery_Distance_tortuosity':artery_t2_list, 'Artery_Squared_curvature_tortuosity':artery_t4_list, 'Artery_Tortuosity_density':artery_t5_list, 'Vein_Fractal_dimension':vein_FD_binary, 'Vein_Vessel_density':vein_VD_binary, 'Vein_Average_width':vein_Average_width,'Vein_Distance_tortuosity':vein_t2_list, 'Vein_Squared_curvature_tortuosity':vein_t4_list, 'Vein_Tortuosity_density':vein_t5_list, 'CRAE_Hubbard':CRAE_Hubbard_list, 'CRVE_Hubbard':CRVE_Hubbard_list, 'AVR_Hubbard':AVR_Hubbard_list, 'CRAE_Knudtson':CRAE_Knudtson_list, 'CRVE_Knudtson':CRVE_Knudtson_list, 'AVR_Knudtson':AVR_Knudtson_list}) -#Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index().drop(columns=['index', 'Name']), Data4stage2] ,axis=1) -Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index(), Data4stage2] ,axis=1) -Data4stage2.to_csv('../../Results/M3/Disc_centred/Disc_Zone_C_Measurement.csv', index = None, encoding='utf8') - - diff --git a/M3_feature_zone/retipy/create_datasets_macular_centred_B.py b/M3_feature_zone/retipy/create_datasets_macular_centred_B.py deleted file mode 100644 index e7e9c6b..0000000 --- a/M3_feature_zone/retipy/create_datasets_macular_centred_B.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 - -# Retipy - Retinal Image Processing on Python -# Copyright (C) 2017 Alejandro Valdes -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -script to estimate the linear tortuosity of a set of retinal images, it will output the values -to a file in the output folder defined in the configuration. The output will only have the -estimated value and it is sorted by image file name. -""" - -import argparse -import glob -# import numpy as np -import os -import h5py -import shutil -import pandas as pd -# import scipy.stats as stats - -from retipy import configuration, retina, tortuosity_measures - - -if os.path.exists('../../Results/M2/artery_vein/macular_Zone_B_centred_artery_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/macular_Zone_B_centred_artery_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/binary_vessel/macular_Zone_B_centred_vein_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/binary_vessel/macular_Zone_B_centred_vein_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/artery_vein/macular_Zone_B_centred_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/macular_Zone_B_centred_binary_skeleton/.ipynb_checkpoints') -if not os.path.exists('../../Results/M3/Macular_centred/Width/'): - os.makedirs('../../Results/M3/Macular_centred/Width/') - -#if os.path.exists('./DDR/av_seg/raw/.ipynb_checkpoints'): -# shutil.rmtree('./DDR/av_seg/raw/.ipynb_checkpoints') - - -parser = argparse.ArgumentParser() - -parser.add_argument( - "-c", - "--configuration", - help="the configuration file location", - default="resources/retipy.config") -args = parser.parse_args() - -CONFIG = configuration.Configuration(args.configuration) -binary_FD_binary,binary_VD_binary,binary_Average_width,binary_t2_list,binary_t4_list,binary_t5_list = [],[],[],[],[],[] -artery_FD_binary,artery_VD_binary,artery_Average_width,artery_t2_list,artery_t4_list,artery_t5_list = [],[],[],[],[],[] -vein_FD_binary,vein_VD_binary,vein_Average_width,vein_t2_list,vein_t4_list,vein_t5_list = [],[],[],[],[],[] -CRAE_Hubbard_list = [] -CRVE_Hubbard_list = [] -AVR_Hubbard_list = [] -CRAE_Knudtson_list = [] -CRVE_Knudtson_list = [] -AVR_Knudtson_list = [] -name_list = [] - -Artery_PATH = '../../Results/M2/artery_vein/macular_Zone_B_centred_artery_skeleton' -Vein_PATH = '../../Results/M2/artery_vein/macular_Zone_B_centred_vein_skeleton' -Binary_PATH = '../../Results/M2/binary_vessel/macular_Zone_B_centred_binary_skeleton' - -for filename in sorted(glob.glob(os.path.join(Binary_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename, store_path='../../Results/M2/binary_vessel/macular_Zone_B_centred_binary_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average, _, _,_,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/binary_vessel/macular_Zone_B_centred_binary_process/') - #print(window.tags) - binary_t2_list.append(t2) - binary_t4_list.append(t4) - binary_t5_list.append(td) - binary_FD_binary.append(FD_binary) - binary_VD_binary.append(VD_binary) - binary_Average_width.append(Average_width) - name_list.append(filename.split('/')[-1]) - - - - - -for filename in sorted(glob.glob(os.path.join(Artery_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/macular_Zone_B_centred_artery_process') - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,CRAE_Hubbard, _,CRAE_Knudtson,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/macular_Zone_B_centred_artery_process/') - #print(window.tags) - artery_t2_list.append(t2) - artery_t4_list.append(t4) - artery_t5_list.append(td) - artery_FD_binary.append(FD_binary) - artery_VD_binary.append(VD_binary) - artery_Average_width.append(Average_width) - CRAE_Hubbard_list.append(CRAE_Hubbard) - CRAE_Knudtson_list.append(CRAE_Knudtson) - - - - -####################################3 - - -for filename in sorted(glob.glob(os.path.join(Vein_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/macular_Zone_B_centred_vein_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,_, CRVE_Hubbard,_,CRVE_Knudtson = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/macular_Zone_B_centred_vein_process/') - #print(window.tags) - vein_t2_list.append(t2) - vein_t4_list.append(t4) - vein_t5_list.append(td) - vein_FD_binary.append(FD_binary) - vein_VD_binary.append(VD_binary) - vein_Average_width.append(Average_width) - CRVE_Hubbard_list.append(CRVE_Hubbard) - CRVE_Knudtson_list.append(CRVE_Knudtson) - - - -AVR_Knudtson_list = [a / b for a,b in zip(CRAE_Knudtson_list, CRVE_Knudtson_list)] -AVR_Hubbard_list = [a / b for a,b in zip(CRAE_Hubbard_list, CRVE_Hubbard_list)] - -Disc_file = pd.read_csv('../../Results/M3/Macular_centred/Disc_cup_results.csv') - -Data4stage2 = pd.DataFrame({'Fractal_dimension':binary_FD_binary, 'Vessel_density':binary_VD_binary, 'Average_width':binary_Average_width,'Distance_tortuosity':binary_t2_list, 'Squared_curvature_tortuosity':binary_t4_list, 'Tortuosity_density':binary_t5_list, 'Artery_Fractal_dimension':artery_FD_binary, 'Artery_Vessel_density':artery_VD_binary, 'Artery_Average_width':artery_Average_width,'Artery_Distance_tortuosity':artery_t2_list, 'Artery_Squared_curvature_tortuosity':artery_t4_list, 'Artery_Tortuosity_density':artery_t5_list, 'Vein_Fractal_dimension':vein_FD_binary, 'Vein_Vessel_density':vein_VD_binary, 'Vein_Average_width':vein_Average_width,'Vein_Distance_tortuosity':vein_t2_list, 'Vein_Squared_curvature_tortuosity':vein_t4_list, 'Vein_Tortuosity_density':vein_t5_list, 'CRAE_Hubbard':CRAE_Hubbard_list, 'CRVE_Hubbard':CRVE_Hubbard_list, 'AVR_Hubbard':AVR_Hubbard_list, 'CRAE_Knudtson':CRAE_Knudtson_list, 'CRVE_Knudtson':CRVE_Knudtson_list, 'AVR_Knudtson':AVR_Knudtson_list}) -#Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index().drop(columns=['index', 'Name']), Data4stage2] ,axis=1) -Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index(), Data4stage2] ,axis=1) -Data4stage2.to_csv('../../Results/M3/Macular_centred/Macular_Zone_B_Measurement.csv', index = None, encoding='utf8') \ No newline at end of file diff --git a/M3_feature_zone/retipy/create_datasets_macular_centred_C.py b/M3_feature_zone/retipy/create_datasets_macular_centred_C.py deleted file mode 100644 index a7286c4..0000000 --- a/M3_feature_zone/retipy/create_datasets_macular_centred_C.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 - -# Retipy - Retinal Image Processing on Python -# Copyright (C) 2017 Alejandro Valdes -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -script to estimate the linear tortuosity of a set of retinal images, it will output the values -to a file in the output folder defined in the configuration. The output will only have the -estimated value and it is sorted by image file name. -""" - -import argparse -import glob -# import numpy as np -import os -import h5py -import shutil -import pandas as pd -# import scipy.stats as stats - -from retipy import configuration, retina, tortuosity_measures - - -if os.path.exists('../../Results/M2/artery_vein/artery_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/artery_binary_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/binary_vessel/binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/binary_vessel/binary_skeleton/.ipynb_checkpoints') -if os.path.exists('../../Results/M2/artery_vein/vein_binary_skeleton/.ipynb_checkpoints'): - shutil.rmtree('../../Results/M2/artery_vein/vein_binary_skeleton/.ipynb_checkpoints') -if not os.path.exists('../../Results/M3/Macular_centred/Width/'): - os.makedirs('../../Results/M3/Macular_centred/Width/') - -#if os.path.exists('./DDR/av_seg/raw/.ipynb_checkpoints'): -# shutil.rmtree('./DDR/av_seg/raw/.ipynb_checkpoints') - - -parser = argparse.ArgumentParser() - -parser.add_argument( - "-c", - "--configuration", - help="the configuration file location", - default="resources/retipy.config") -args = parser.parse_args() - -CONFIG = configuration.Configuration(args.configuration) -binary_FD_binary,binary_VD_binary,binary_Average_width,binary_t2_list,binary_t4_list,binary_t5_list = [],[],[],[],[],[] -artery_FD_binary,artery_VD_binary,artery_Average_width,artery_t2_list,artery_t4_list,artery_t5_list = [],[],[],[],[],[] -vein_FD_binary,vein_VD_binary,vein_Average_width,vein_t2_list,vein_t4_list,vein_t5_list = [],[],[],[],[],[] -CRAE_Hubbard_list = [] -CRVE_Hubbard_list = [] -AVR_Hubbard_list = [] -CRAE_Knudtson_list = [] -CRVE_Knudtson_list = [] -AVR_Knudtson_list = [] -name_list = [] - -Artery_PATH = '../../Results/M2/artery_vein/macular_Zone_C_centred_artery_skeleton' -Vein_PATH = '../../Results/M2/artery_vein/macular_Zone_C_centred_vein_skeleton' -Binary_PATH = '../../Results/M2/binary_vessel/macular_Zone_C_centred_binary_skeleton' - -for filename in sorted(glob.glob(os.path.join(Binary_PATH, '*.png'))): - segmentedImage = retina.Retina(None, filename, store_path='../../Results/M2/binary_vessel/macular_Zone_C_centred_binary_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average, _, _,_,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/binary_vessel/macular_Zone_C_centred_binary_process/') - #print(window.tags) - binary_t2_list.append(t2) - binary_t4_list.append(t4) - binary_t5_list.append(td) - binary_FD_binary.append(FD_binary) - binary_VD_binary.append(VD_binary) - binary_Average_width.append(Average_width) - name_list.append(filename.split('/')[-1]) - - - - - -for filename in sorted(glob.glob(os.path.join(Artery_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/macular_Zone_C_centred_artery_process') - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,CRAE_Hubbard, _,CRAE_Knudtson,_ = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/macular_Zone_C_centred_artery_process/') - #print(window.tags) - artery_t2_list.append(t2) - artery_t4_list.append(t4) - artery_t5_list.append(td) - artery_FD_binary.append(FD_binary) - artery_VD_binary.append(VD_binary) - artery_Average_width.append(Average_width) - CRAE_Hubbard_list.append(CRAE_Hubbard) - CRAE_Knudtson_list.append(CRAE_Knudtson) - - - - -####################################3 - - -for filename in sorted(glob.glob(os.path.join(Vein_PATH, '*.png'))): - - segmentedImage = retina.Retina(None, filename,store_path='../../Results/M2/artery_vein/macular_Zone_C_centred_vein_process') - #segmentedImage.threshold_image() - #segmentedImage.reshape_square() - #window_sizes = segmentedImage.get_window_sizes() - window_sizes = [912] - window = retina.Window( - segmentedImage, window_sizes[-1], min_pixels=CONFIG.pixels_per_window) - FD_binary,VD_binary,Average_width,t2, t4, td, vessel_count_list, w1_list, w1_list_average,_, CRVE_Hubbard,_,CRVE_Knudtson = tortuosity_measures.evaluate_window(window, CONFIG.pixels_per_window, CONFIG.sampling_size, CONFIG.r_2_threshold,store_path='../../Results/M2/artery_vein/macular_Zone_C_centred_vein_process/') - #print(window.tags) - vein_t2_list.append(t2) - vein_t4_list.append(t4) - vein_t5_list.append(td) - vein_FD_binary.append(FD_binary) - vein_VD_binary.append(VD_binary) - vein_Average_width.append(Average_width) - CRVE_Hubbard_list.append(CRVE_Hubbard) - CRVE_Knudtson_list.append(CRVE_Knudtson) - - - -AVR_Knudtson_list = [a / b for a,b in zip(CRAE_Knudtson_list, CRVE_Knudtson_list)] -AVR_Hubbard_list = [a / b for a,b in zip(CRAE_Hubbard_list, CRVE_Hubbard_list)] - -Disc_file = pd.read_csv('../../Results/M3/Macular_centred/Disc_cup_results.csv') - -Data4stage2 = pd.DataFrame({'Fractal_dimension':binary_FD_binary, 'Vessel_density':binary_VD_binary, 'Average_width':binary_Average_width,'Distance_tortuosity':binary_t2_list, 'Squared_curvature_tortuosity':binary_t4_list, 'Tortuosity_density':binary_t5_list, 'Artery_Fractal_dimension':artery_FD_binary, 'Artery_Vessel_density':artery_VD_binary, 'Artery_Average_width':artery_Average_width,'Artery_Distance_tortuosity':artery_t2_list, 'Artery_Squared_curvature_tortuosity':artery_t4_list, 'Artery_Tortuosity_density':artery_t5_list, 'Vein_Fractal_dimension':vein_FD_binary, 'Vein_Vessel_density':vein_VD_binary, 'Vein_Average_width':vein_Average_width,'Vein_Distance_tortuosity':vein_t2_list, 'Vein_Squared_curvature_tortuosity':vein_t4_list, 'Vein_Tortuosity_density':vein_t5_list, 'CRAE_Hubbard':CRAE_Hubbard_list, 'CRVE_Hubbard':CRVE_Hubbard_list, 'AVR_Hubbard':AVR_Hubbard_list, 'CRAE_Knudtson':CRAE_Knudtson_list, 'CRVE_Knudtson':CRVE_Knudtson_list, 'AVR_Knudtson':AVR_Knudtson_list}) -#Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index().drop(columns=['index', 'Name']), Data4stage2] ,axis=1) -Data4stage2 = pd.concat([Disc_file[Disc_file['Name'].isin(name_list)].reset_index(), Data4stage2] ,axis=1) -Data4stage2.to_csv('../../Results/M3/Macular_centred/Macular_Zone_C_Measurement.csv', index = None, encoding='utf8') \ No newline at end of file diff --git a/app/lambda_predict.py b/app/lambda_predict.py new file mode 100644 index 0000000..b38e9af --- /dev/null +++ b/app/lambda_predict.py @@ -0,0 +1,101 @@ +from curses import BUTTON1_CLICKED +from io import BytesIO +import boto3 +import logging +from glob import glob +from urllib.parse import unquote_plus +import json +import os + +import automorph.M0_Preprocess.EyeQ_process_main as M0_EQ +import automorph.M1_Retinal_Image_quality_EyePACS.test_outside as M1_EP +import automorph.M1_Retinal_Image_quality_EyePACS.merge_quality_assessment as M1_QA +import automorph.M2_Vessel_seg.test_outside_integrated as M2_VS +import automorph.M2_Artery_vein.test_outside as M2_AV +import automorph.M2_lwnet_disc_cup.generate_av_results as M2_DC +import automorph.M3_feature_whole_pic.retipy.create_datasets_macular_centred as CDMC +import automorph.M3_feature_zone.retipy.create_datasets_disc_centred_B as CDDCB +import automorph.M3_feature_zone.retipy.create_datasets_disc_centred_C as CDDCC +import automorph.M3_feature_zone.retipy.create_datasets_macular_centred_B as CDMCB +import automorph.M3_feature_zone.retipy.create_datasets_macular_centred_C as CDMCC +import automorph.config as gv +import imghdr + +# Define logger class +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# get s3 client +s3 = boto3.client('s3', region_name='us-west-2') + +def lambda_handler(event,context): + + record = event['Records'][0] # records [0] + bucket = record['s3']['bucket']['name'] + key = unquote_plus(record['s3']['object']['key']) + tmpkey = key.split('/')[-1] + download_path = gv.image_dir + if not os.path.exists(download_path): + os.mkdir(download_path) + s3.download_file(bucket, key, download_path+tmpkey) + logger.info("object download from s3") + + # quick png check + ftype = imghdr.what(download_path+tmpkey) + if ftype != 'png': + logger.info("{} is not a png file, aborting".format(key)) + return {'statusCode': 200, 'headers':{'Content-type':'application/json'}, + 'body': "uploaded object {} was not a png filetype".format(key) + } + + # Pre-processing + logger.info('Pre-processing') + M0_EQ.EyeQ_process() + + # Eye Quality + M1_EP.M1_image_quality() + M1_QA.quality_assessment() + + #M2 stages + logger.info("starting_vessel_seg") + M2_VS.M2_vessel_seg() + logger.info("starting AV seg") + M2_AV.M2_artery_vein() + logger.info("starting disc cup seg") + M2_DC.M2_disc_cup() + + CDDCB.create_data_disc_centred_B() + + CDDCC.create_data_disc_centred_C() + + CDMCB.create_macular_centred_B() + + CDMCC.create_macular_centred_C() + + CDMC.create_dataset_macular_centred() + logger.info("finished pipeline") + + logger.info("copying files") + + output_bucket = "eyeact-automorph2" + sub_dir = tmpkey.split('.png')[0] + + for root, dir, files in os.walk(gv.results_dir): + logger.info("Root: {}, dir {}, files {}".format(root, dir, files)) + for f in files: + png = os.path.join(root, f) + id = os.path.join(sub_dir, root.split(gv.results_dir)[-1], f) # need to change for lee lab install + logger.info("printing at {}".format(id)) + s3.upload_file(png, output_bucket, id) + + return { + 'statusCode': 200, + 'headers':{ + 'Content-type':'application/json' + }, + 'body': "finished" + } + +#with open("/data/anand/AutoMorph_Lee/test_event.json", 'r') as f: + #data = json.load(f) +#lambda_handler(data, 1) diff --git a/automorph/M0_Preprocess/EyeQ_process_main.py b/automorph/M0_Preprocess/EyeQ_process_main.py new file mode 100644 index 0000000..f35ddf0 --- /dev/null +++ b/automorph/M0_Preprocess/EyeQ_process_main.py @@ -0,0 +1,91 @@ +from genericpath import isfile +from glob import glob +import pandas as pd +import os +from PIL import ImageFile +ImageFile.LOAD_TRUNCATED_IMAGES = True +from .fundus_prep import process_without_gb, imread, imwrite +from random import sample +from pathlib import Path + + +def create_resolution_information(image_dir): + # create resolution of 1 for all images if information not known. + + images = glob("{}*.png".format(image_dir)) + print("{} images found with glob".format(len(images))) + + res_csv_pth = Path(__file__).parent / "../resolution_information.csv" + with open(res_csv_pth, "w") as f: + f.write("fundus,res\n") + f.writelines("{},1\n".format(x.split('/')[-1]) for x in images) + +def process(image_list, save_path, cfg): + """ Crops each image in the image list to create the smallest square that fits all retinal colored information and + removes background retina pixels + """ + + radius_list = [] + centre_list_w = [] + centre_list_h = [] + name_list = [] + list_resolution = [] + scale_resolution = [] + + resolution_csv_path = Path(__file__).parent / "../resolution_information.csv" + if not os.path.exists(resolution_csv_path): + create_resolution_information(cfg.image_dir) + resolution_list = pd.read_csv(resolution_csv_path) + + for image_path in image_list: + + dst_image = cfg.image_dir + image_path + if os.path.exists('{}M0/images/'.format(save_path) + image_path): + print('continue...') + continue + try: + if len(resolution_list['res'][resolution_list['fundus']==image_path].values) == 0: + resolution_ = 1 + else: + resolution_ = resolution_list['res'][resolution_list['fundus']==image_path].values[0] + list_resolution.append(resolution_) + + img = imread(dst_image) + r_img, borders, mask, label, radius_list,centre_list_w, centre_list_h = process_without_gb(img,img,radius_list,centre_list_w, centre_list_h) + + if not cfg.sparse: + imwrite(save_path + image_path.split('.')[0] + '.png', r_img) + + except IndexError: + print("\nThe file {} has not been added to the resolution_information.csv found at {}\n\ + Please update this file with the script found at /lee_lab_scripts/create_resolution.py and re-run the code".format( \ + image_path, resolution_csv_path)) + exit(1) + except ValueError: + print('error with boundary for') + + name_list.append(cfg.image_dir+image_path) + + + scale_list = [a*2/912 for a in radius_list] + + scale_resolution = [a*b*1000 for a,b in zip(list_resolution,scale_list)] + + Data4stage2 = pd.DataFrame({'Name':name_list, 'centre_w':centre_list_w, 'centre_h':centre_list_h, 'radius':radius_list, 'Scale':scale_list, 'Scale_resolution':scale_resolution}) + Data4stage2.to_csv('{}M0/crop_info.csv'.format(cfg.results_dir), index = None, encoding='utf8') + + +def EyeQ_process(cfg): + + if cfg.sample_num: + print("Sampling {} images from {}".format(cfg.sample_num, cfg.image_dir)) + image_list = sample(sorted(os.listdir(cfg.image_dir)), cfg.sample_num) + else: + image_list = sorted(os.listdir(cfg.image_dir)) + + save_path = cfg.results_dir + "M0/images/" + + if not os.path.exists('{}'.format(save_path)): + os.makedirs('{}'.format(save_path)) + + process(image_list, save_path, cfg) \ No newline at end of file diff --git a/M0_Preprocess/__init__.py b/automorph/M0_Preprocess/__init__.py similarity index 100% rename from M0_Preprocess/__init__.py rename to automorph/M0_Preprocess/__init__.py diff --git a/M0_Preprocess/fundus_prep.py b/automorph/M0_Preprocess/fundus_prep.py similarity index 72% rename from M0_Preprocess/fundus_prep.py rename to automorph/M0_Preprocess/fundus_prep.py index 1a5726c..ee5b65f 100644 --- a/M0_Preprocess/fundus_prep.py +++ b/automorph/M0_Preprocess/fundus_prep.py @@ -1,6 +1,8 @@ +from cv2 import threshold import numpy as np import os import cv2 +from scipy.ndimage import label def imread(file_path, c=None): @@ -10,10 +12,11 @@ def imread(file_path, c=None): im = cv2.imread(file_path, c) if im is None: - raise 'Can not read image' + print('Can not read image at filepath: {}'.format(file_path)) + raise "error" if im.ndim == 3 and im.shape[2] == 3: - im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) + im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) #cv2 reads images in bgr format return im @@ -39,7 +42,7 @@ def get_mask_BZ(img): #cv2.waitKey() #print(threhold) _, mask = cv2.threshold(gray_img, max(5,threhold), 1, cv2.THRESH_BINARY) - + #cv2.imshow('bz_mask', mask*255) #cv2.waitKey() nn_mask = np.zeros((mask.shape[0]+2,mask.shape[1]+2),np.uint8) @@ -65,6 +68,9 @@ def _get_center_by_edge(mask): def _get_radius_by_mask_center(mask,center): + """ apply gradient (difference between erosion and dilation) to the mask to find the border, then calculate the radius from this border. Find the + distance between each point in the mask and the centre then select the radius as the distance slightly larger than 99.5% of the largest distance + """ mask=mask.astype(np.uint8) ksize=max(mask.shape[1]//400*2+1,3) kernel=cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ksize,ksize)) @@ -73,8 +79,12 @@ def _get_radius_by_mask_center(mask,center): index=np.where(mask>0) d_int=np.sqrt((index[0]-center[0])**2+(index[1]-center[1])**2) b_count=np.bincount(np.ceil(d_int).astype(np.int)) - radius=np.where(b_count>b_count.max()*0.995)[0].max() - return radius + if len(b_count) == 0: # if there are no unique values in the bin + radius = 0 + return radius + else: + radius = np.where(b_count>b_count.max()*0.995)[0].max() + return radius def _get_circle_by_center_bbox(shape,center,bbox,radius): @@ -86,6 +96,29 @@ def _get_circle_by_center_bbox(shape,center,bbox,radius): # center_mask[bbox[0]:min(bbox[0]+bbox[2],center_mask.shape[0]),bbox[1]:min(bbox[1]+bbox[3],center_mask.shape[1])]=tmp_mask return center_mask +def get_largest_element(mask): + """ + Uses structuring element in scipy ndimage label to find all connected features in an array and only keep the + feature with the most elements. + + Params: + mask (np.array) binary mask + Returns: + largest_mask (np.array) binary mask only containing the largest connected element of the input mask + """ + + l_mask, n_labels = label(mask) + + if n_labels > 0: + label_dict = {} + for l in range(1,n_labels+1): + label_dict[l] = np.sum(l_mask == l) + sorted_label = sorted(label_dict.items(), key= lambda x: x[1], reverse=True) + largest_mask = l_mask == sorted_label[0][0] + return largest_mask + else: + largest_mask = mask + return largest_mask def get_mask(img): if img.ndim ==3: @@ -102,16 +135,33 @@ def get_mask(img): #g_img = cv2.resize(g_img,(0,0),fx = 0.5,fy = 0.5) tg_img=cv2.normalize(g_img, None, 0, 255, cv2.NORM_MINMAX) tmp_mask=get_mask_BZ(tg_img) + tmp_mask = get_largest_element(tmp_mask) center=_get_center_by_edge(tmp_mask) + #bbox=_get_bbox_by_mask(tmp_mask) #print(center) #cv2.imshow('ImageWindow', tmp_mask*255) #cv2.waitKey() radius=_get_radius_by_mask_center(tmp_mask,center) + #resize back #center = [center[0]*2,center[1]*2] #radius = int(radius*2) + if radius == 0: # if there is no radius then this cant work just return zeros + return tmp_mask, (0,0,0,0), [0,0], radius + # if the radius is greater than either the height or width of the image + if ((radius +center[0]) > h) or ((center[0] - radius) <= 0): + if center[0] < (h/2): + radius = center[0] + else: + radius = h-center[0] + elif ((radius +center[1]) > w) or ((center[1] - radius) <= 0): + if center[1]< (w/2): + radius = center[1] + else: + radius = w-center[1] + center = [center[0], center[1]] radius = int(radius) s_h = max(0,int(center[0] - radius)) @@ -162,10 +212,10 @@ def process_without_gb(img, label,radius_list,centre_list_w, centre_list_h): # result_img: preprocessed image # borders: remove border, supplement mask # mask: mask for preprocessed image + + #return null if image is all zeros borders = [] mask, bbox, center, radius = get_mask(img) - #print('center is: ',center) - #print('radius is: ',radius) r_img = mask_image(img, mask) r_img, r_border = remove_back_area(r_img,bbox=bbox) mask, _ = remove_back_area(mask,border=r_border) @@ -180,5 +230,10 @@ def process_without_gb(img, label,radius_list,centre_list_w, centre_list_h): radius_list.append(radius) centre_list_w.append(int(center[0])) centre_list_h.append(int(center[1])) + + # return input image in r_img is zero dimension + if len(r_img) == 0: + r_img = img + return r_img,borders,(mask*255).astype(np.uint8),label, radius_list,centre_list_w, centre_list_h diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/0_seed_42/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/0_seed_42/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/0_seed_42/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/0_seed_42/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/1_seed_40/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/1_seed_40/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/1_seed_40/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/1_seed_40/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/2_seed_38/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/2_seed_38/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/2_seed_38/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/2_seed_38/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/3_seed_36/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/3_seed_36/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/3_seed_36/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/3_seed_36/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/4_seed_34/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/4_seed_34/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/4_seed_34/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/4_seed_34/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/5_seed_32/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/5_seed_32/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/5_seed_32/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/5_seed_32/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/6_seed_30/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/6_seed_30/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/6_seed_30/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/6_seed_30/best_loss_checkpoint.pth diff --git a/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/7_seed_28/best_loss_checkpoint.pth b/automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/7_seed_28/best_loss_checkpoint.pth similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/7_seed_28/best_loss_checkpoint.pth rename to automorph/M1_Retinal_Image_quality_EyePACS/Retinal_quality/EyePACS_quality/efficientnet/7_seed_28/best_loss_checkpoint.pth diff --git a/automorph/M1_Retinal_Image_quality_EyePACS/__init__.py b/automorph/M1_Retinal_Image_quality_EyePACS/__init__.py new file mode 100644 index 0000000..303a3e4 --- /dev/null +++ b/automorph/M1_Retinal_Image_quality_EyePACS/__init__.py @@ -0,0 +1,2 @@ +#from pkg_resources import resource_filename +#filepath = resource_filename("M1_Retinal_Image_quality_EyePACS", 'Retinal_quality') diff --git a/M1_Retinal_Image_quality_EyePACS/dataset.py b/automorph/M1_Retinal_Image_quality_EyePACS/dataset.py similarity index 81% rename from M1_Retinal_Image_quality_EyePACS/dataset.py rename to automorph/M1_Retinal_Image_quality_EyePACS/dataset.py index 00bcfe2..4d33196 100644 --- a/M1_Retinal_Image_quality_EyePACS/dataset.py +++ b/automorph/M1_Retinal_Image_quality_EyePACS/dataset.py @@ -91,25 +91,42 @@ def __getitem__(self, index): class BasicDataset_OUT(Dataset): 'Characterizes a dataset for PyTorch' - def __init__(self, image_dir, image_size, n_classes, train_or): + def __init__(self, image_dir, image_size, n_classes, train_or, crop_csv): 'Initialization' self.image_size = image_size self.image_dir = image_dir self.n_classes = n_classes self.train_or = train_or - - self.ids = [splitext(file)[0] for file in sorted(listdir(image_dir)) - if not file.startswith('.')] - logging.info(f'Creating dataset with {len(self.ids)} examples') + self.crop_csv = crop_csv + fps = pd.read_csv(crop_csv, usecols=['Name']).values.flatten() + self.file_paths = fps + + # create the dataset by accessing the crop info and loading the + + logging.info(f'Creating dataset with {len(self.file_paths)} examples') def __len__(self): 'Denotes the total number of samples' - return len(self.ids) + return len(self.file_paths) @classmethod - def preprocess(self, pil_img, img_size, train_or,index): + def preprocess(self, pil_img, img_size, train_or, img_path, crop_csv): #w, h = pil_img.size + + # crop the image based on the pre-poc csv at M0 + df = pd.read_csv(crop_csv) + row = df[df['Name'] == img_path] + + c_w = row['centre_w'] + c_h = row['centre_h'] + r = row['radius'] + w_min, w_max = int(c_w-r), int(c_w+r) + h_min, h_max = int(c_h-r), int(c_h+r) + + pil_img = pil_img.crop((h_min, w_min, h_max, w_max)) + #pil_img.save("/data/anand/test_cropped.png") #TODO remove + newW, newH = img_size[0], img_size[1] assert newW > 0 and newH > 0, 'Scale is too small' pil_img = pil_img.resize((newW, newH)) @@ -118,7 +135,6 @@ def preprocess(self, pil_img, img_size, train_or,index): mean_value=np.mean(img_array[img_array > 0]) std_value=np.std(img_array[img_array > 0]) img_array=(img_array-mean_value)/std_value - #print(np.unique(img_array)) if len(img_array.shape) == 2: img_array = np.expand_dims(img_array, axis=2) @@ -127,14 +143,13 @@ def preprocess(self, pil_img, img_size, train_or,index): img_array = img_array.transpose((2, 0, 1)) return img_array - + def __getitem__(self, index): 'Generates one sample of data' # Select sample - idx = self.ids[index] - img_file = glob(self.image_dir + idx + '.*') - image = Image.open(img_file[0]) - image_processed = self.preprocess(image, self.image_size, self.train_or, index) + img_file = self.file_paths[index] + image = Image.open(img_file) + image_processed = self.preprocess(image, self.image_size, self.train_or, img_file, self.crop_csv) return { 'img_file': img_file, diff --git a/M1_Retinal_Image_quality_EyePACS/eval.py b/automorph/M1_Retinal_Image_quality_EyePACS/eval.py similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/eval.py rename to automorph/M1_Retinal_Image_quality_EyePACS/eval.py diff --git a/automorph/M1_Retinal_Image_quality_EyePACS/merge_quality_assessment.py b/automorph/M1_Retinal_Image_quality_EyePACS/merge_quality_assessment.py new file mode 100644 index 0000000..912e86f --- /dev/null +++ b/automorph/M1_Retinal_Image_quality_EyePACS/merge_quality_assessment.py @@ -0,0 +1,69 @@ +from glob import glob +import numpy as np +import pandas as pd +import shutil +import os + +def quality_assessment(cfg): + + result_Eyepacs_path = '{}M1/results_ensemble.csv'.format(cfg.results_dir) #TODO replace this in some way? + + if not os.path.exists('{}M1/Good_quality/'.format(cfg.results_dir)): + os.makedirs('{}M1/Good_quality/'.format(cfg.results_dir)) + if not os.path.exists('{}M1/Bad_quality/'.format(cfg.results_dir)): + os.makedirs('{}M1/Bad_quality/'.format(cfg.results_dir)) + + result_Eyepacs_ = pd.read_csv(result_Eyepacs_path) + + # save the Good_quality + gq = result_Eyepacs_[result_Eyepacs_['Prediction'] == 0]['Name'].values + if cfg.quality_thresh == 'good': + usable = result_Eyepacs_[(result_Eyepacs_['Prediction'] == 1) & + (result_Eyepacs_['softmax_bad'] < 0.25)]['Name'].values + if cfg.quality_thresh == 'usable': + usable = result_Eyepacs_[(result_Eyepacs_['Prediction'] == 1)]['Name'].values + if cfg.quality_thresh == 'all': + usable = result_Eyepacs_[result_Eyepacs_['Prediction'].isin([1,2])]['Name'].values + gq = np.append(gq, usable) + gq = list(gq) + + # find the bad quality images + bq = result_Eyepacs_[~result_Eyepacs_['Name'].isin(gq)]['Name'].values + + # drop all images where radius and centre are 0 + def find_zeros(a): + if (a['centre_w'] == 0) & (a['centre_h'] == 0) & (a['radius'] == 0): + return False + else: + return True + + # merge the images with the crop data and save them as a csv + crops = pd.read_csv(cfg.results_dir+'M0/crop_info.csv', usecols=['Name', 'centre_h', 'centre_w', 'radius']) + crops = crops[crops.apply(find_zeros, axis=1)] + gq_crops = crops[crops['Name'].isin(gq)] + + gq_crops.to_csv('{}/M1/Good_quality/image_list.csv'.format(cfg.results_dir), index=False) + bq_crops = crops[crops['Name'].isin(bq)] + bq_crops.to_csv('{}/M1/Bad_quality/image_list.csv'.format(cfg.results_dir), index=False) + + # print metrics + print('Gradable cases by EyePACS_QA is {} '.format(len(gq))) + print('Ungradable cases by EyePACS_QA is {} '.format(len(bq))) + + + # if not sparse, then copy and save all the images into a M1/Good_quality or M1/Bad_quality folder + if not cfg.sparse: + print('copying good and bad quality images from M0/images into subsequent directories') + + for id in gq: + + f = id.split('/')[-1].split('.')[0]+'.png' + name = cfg.results_dir+"M0/images/"+f + shutil.copy(name, '{}M1/Good_quality/{}'.format(cfg.results_dir, f)) + + for id in bq: + + f = id.split('/')[-1].split('.')[0]+'.png' + name = cfg.results_dir+"M0/images/"+f + shutil.copy(name, '{}M1/Bad_quality/{}'.format(cfg.results_dir, f)) + \ No newline at end of file diff --git a/M1_Retinal_Image_quality_EyePACS/model.py b/automorph/M1_Retinal_Image_quality_EyePACS/model.py similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/model.py rename to automorph/M1_Retinal_Image_quality_EyePACS/model.py diff --git a/M1_Retinal_Image_quality_EyePACS/pytorchtools.py b/automorph/M1_Retinal_Image_quality_EyePACS/pytorchtools.py similarity index 100% rename from M1_Retinal_Image_quality_EyePACS/pytorchtools.py rename to automorph/M1_Retinal_Image_quality_EyePACS/pytorchtools.py diff --git a/M1_Retinal_Image_quality_EyePACS/test_outside.py b/automorph/M1_Retinal_Image_quality_EyePACS/test_outside.py similarity index 78% rename from M1_Retinal_Image_quality_EyePACS/test_outside.py rename to automorph/M1_Retinal_Image_quality_EyePACS/test_outside.py index f0b51b9..78540bd 100644 --- a/M1_Retinal_Image_quality_EyePACS/test_outside.py +++ b/automorph/M1_Retinal_Image_quality_EyePACS/test_outside.py @@ -1,4 +1,3 @@ - ''' Code of testing ''' @@ -7,6 +6,7 @@ import os import sys import csv +from tabnanny import check import time import torch import numpy as np @@ -16,9 +16,10 @@ from pycm import * import matplotlib import matplotlib.pyplot as plt -from dataset import BasicDataset_OUT +from .dataset import BasicDataset_OUT from torch.utils.data import DataLoader -from model import Resnet101_fl, InceptionV3_fl, Densenet161_fl, Resnext101_32x8d_fl, MobilenetV2_fl, Vgg16_bn_fl, Efficientnet_fl +from .model import Resnet101_fl, InceptionV3_fl, Densenet161_fl, Resnext101_32x8d_fl, MobilenetV2_fl, Vgg16_bn_fl, Efficientnet_fl +from pathlib import Path #torch.distributed.init_process_group(backend="nccl") @@ -37,7 +38,9 @@ def test_net(model_fl_1, model_fl_7, model_fl_8, test_dir, + args, device, + cfg, epochs=5, batch_size=20, image_size=(512,512), @@ -45,19 +48,15 @@ def test_net(model_fl_1, ): since = time.time() - storage_path ="Ensemble_exp_{}/{}/train_on_{}/test_on_{}/".format(args.task, args.load, args.model, args.dataset) #dir_mask = "./data/{}/training/1st_manual/".format(args.dataset) dataset_name = args.dataset n_classes = args.n_class # create files - if not os.path.isdir(storage_path): - os.makedirs(storage_path) - - dataset = BasicDataset_OUT(test_dir, image_size, n_classes, train_or=False) + dataset = BasicDataset_OUT(test_dir, image_size, n_classes, train_or=False, crop_csv=cfg.results_dir+'M0/crop_info.csv') n_test = len(dataset) - val_loader = DataLoader(dataset, batch_size, shuffle=False, num_workers=16, pin_memory=False, drop_last=False) + val_loader = DataLoader(dataset, batch_size, shuffle=False, num_workers=cfg.worker, pin_memory=False, drop_last=False) prediction_decode_list = [] filename_list = [] @@ -79,7 +78,7 @@ def test_net(model_fl_1, with tqdm(total=n_test, desc=f'Epoch {epoch + 1}/{epochs}', unit='img') as pbar: for batch in val_loader: imgs = batch['image'] - filename = batch['img_file'][0] + filename = batch['img_file'] mask_pred_tensor_small_all = 0 imgs = imgs.to(device=device, dtype=torch.float32) ##################sigmoid or softmax @@ -148,10 +147,9 @@ def test_net(model_fl_1, pbar.update(imgs.shape[0]) Data4stage2 = pd.DataFrame({'Name':filename_list, 'softmax_good':np.array(prediction_list_mean)[:,0],'softmax_usable':np.array(prediction_list_mean)[:,1],'softmax_bad':np.array(prediction_list_mean)[:,2], 'good_sd':np.array(prediction_list_std)[:,0],'usable_sd':np.array(prediction_list_std)[:,1],'bad_sd':np.array(prediction_list_std)[:,2], 'Prediction': prediction_decode_list}) - Data4stage2.to_csv('./test_outside/results_ensemble.csv', index = None, encoding='utf8') - if not os.path.exists('../Results/M1'): - os.makedirs('../Results/M1') - Data4stage2.to_csv('../Results/M1/results_ensemble.csv', index = None, encoding='utf8') + if not os.path.exists('{}M1'.format(cfg.results_dir)): + os.makedirs('{}M1'.format(cfg.results_dir)) + Data4stage2.to_csv('{}M1/results_ensemble.csv'.format(cfg.results_dir), index = None, encoding='utf8') @@ -183,16 +181,31 @@ def get_args(): return parser.parse_args() - -if __name__ == '__main__': +class M1_get_args(): + + def __init__(self, cfg): + + self.epochs = 1 + self.batchsize = cfg.batch_size + self.load = "EyePACS_quality" + self.test_dir = cfg.image_dir + self.n_class = 3 + self.dataset = "customised_data" + self.task = "Retinal_quality" + self.round = 0 + self.model = "efficientnet" + self.seed = 42 + self.local_rank = 0 + + +def M1_image_quality(cfg): logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - args = get_args() + args = M1_get_args(cfg) - torch.cuda.set_device(args.local_rank) - device = torch.device("cuda", args.local_rank) + device = torch.device(cfg.device) - #logging.info(f'Using device {device}') + logging.info(f'Using device {device}') test_dir = args.test_dir dataset=args.dataset @@ -227,15 +240,6 @@ def get_args(): if args.model == 'vgg16bn': model_fl = Vgg16_bn_fl(pretrained=True) - checkpoint_path_1 = './{}/{}/{}/7_seed_28/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_2 = './{}/{}/{}/6_seed_30/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_3 = './{}/{}/{}/5_seed_32/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_4 = './{}/{}/{}/4_seed_34/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_5 = './{}/{}/{}/3_seed_36/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_6 = './{}/{}/{}/2_seed_38/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_7 = './{}/{}/{}/1_seed_40/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - checkpoint_path_8 = './{}/{}/{}/0_seed_42/best_loss_checkpoint.pth'.format(args.task, args.load, args.model ) - model_fl_1.to(device=device) model_fl_2.to(device=device) model_fl_3.to(device=device) @@ -245,33 +249,45 @@ def get_args(): model_fl_7.to(device=device) model_fl_8.to(device=device) - map_location = {'cuda:%d' % 0: 'cuda:%d' % args.local_rank} + checkpoint_path_1 = str(Path(__file__).parent / './{}/{}/{}/7_seed_28/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_2 = str(Path(__file__).parent / './{}/{}/{}/6_seed_30/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_3 = str(Path(__file__).parent / './{}/{}/{}/5_seed_32/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_4 = str(Path(__file__).parent / './{}/{}/{}/4_seed_34/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_5 = str(Path(__file__).parent / './{}/{}/{}/3_seed_36/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_6 = str(Path(__file__).parent / './{}/{}/{}/2_seed_38/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_7 = str(Path(__file__).parent / './{}/{}/{}/1_seed_40/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + checkpoint_path_8 = str(Path(__file__).parent / './{}/{}/{}/0_seed_42/best_loss_checkpoint.pth'.format(args.task, args.load, args.model )) + + logging.info("path {}".format(checkpoint_path_1)) + logging.info("exists {}".format(os.path.exists(checkpoint_path_1))) + if args.load: model_fl_1.load_state_dict( - torch.load(checkpoint_path_1, map_location="cuda:0") + torch.load(checkpoint_path_1, map_location=device) # can add maplocation if I want here ) model_fl_2.load_state_dict( - torch.load(checkpoint_path_2, map_location="cuda:0") + torch.load(checkpoint_path_2, map_location=device) ) model_fl_3.load_state_dict( - torch.load(checkpoint_path_3, map_location="cuda:0") + torch.load(checkpoint_path_3, map_location=device) ) model_fl_4.load_state_dict( - torch.load(checkpoint_path_4, map_location="cuda:0") + torch.load(checkpoint_path_4, map_location=device) ) model_fl_5.load_state_dict( - torch.load(checkpoint_path_5, map_location="cuda:0") + torch.load(checkpoint_path_5, map_location=device) ) model_fl_6.load_state_dict( - torch.load(checkpoint_path_6, map_location="cuda:0") + torch.load(checkpoint_path_6, map_location=device) ) model_fl_7.load_state_dict( - torch.load(checkpoint_path_7, map_location="cuda:0") + torch.load(checkpoint_path_7, map_location=device) ) model_fl_8.load_state_dict( - torch.load(checkpoint_path_8, map_location="cuda:0") + torch.load(checkpoint_path_8, map_location=device) ) + # faster convolutions, but more memory # cudnn.benchmark = True @@ -285,7 +301,9 @@ def get_args(): model_fl_7, model_fl_8, test_dir, - device=device, + args, + device, + cfg, epochs=args.epochs, batch_size=args.batchsize, image_size=img_size) @@ -297,9 +315,3 @@ def get_args(): except SystemExit: os._exit(0) - - - - - - diff --git a/automorph/M1_Retinal_Image_quality_EyePACS/test_outside.sh b/automorph/M1_Retinal_Image_quality_EyePACS/test_outside.sh new file mode 100644 index 0000000..518d335 --- /dev/null +++ b/automorph/M1_Retinal_Image_quality_EyePACS/test_outside.sh @@ -0,0 +1,34 @@ + +CUDA_NUMBER="" + +export PYTHONPATH=.:$PYTHONPATH + +for model in 'efficientnet' +#for model in 'densenet169' +do + for n_round in 0 + do + seed_number=$((42-2*n_round)) + CUDA_VISIBLE_DEVICES=${CUDA_NUMBER} python test_outside.py --e=1 --b=1 --task_name='Retinal_quality' --model=${model} --round=${n_round} --train_on_dataset='EyePACS_quality' \ + --test_on_dataset='customised_data' --test_csv_dir='../Results/M0/images/' --n_class=3 --seed_num=${seed_number} + + + done + +done + + + +M1_args = { + "epochs":1, + "batchsize":1, + "load": False, + "test_dir": ???, # Results/M0/images + "n_class": False, + "dataset": "customised_data", + "task": "Retinal_quality", + "round": 0, + "model": "efficientnet", + "seed": 42, + "local_rank": 0 +} \ No newline at end of file diff --git a/automorph/M1_Retinal_Image_quality_EyePACS/test_outside/results_ensemble.csv b/automorph/M1_Retinal_Image_quality_EyePACS/test_outside/results_ensemble.csv new file mode 100644 index 0000000..3ffca69 --- /dev/null +++ b/automorph/M1_Retinal_Image_quality_EyePACS/test_outside/results_ensemble.csv @@ -0,0 +1,21 @@ +Name,softmax_good,softmax_usable,softmax_bad,good_sd,usable_sd,bad_sd,Prediction +../Results/M0/images/0.png,0.08458339,0.15188871,0.76352787,0.0888564,0.104990885,0.18334606,2 +../Results/M0/images/1.png,0.04542862,0.09459405,0.8599773,0.06297997,0.07608976,0.1312824,2 +../Results/M0/images/10.png,0.23822345,0.25255758,0.509219,0.21948966,0.09300513,0.28570354,2 +../Results/M0/images/11.png,0.2654615,0.26674983,0.46778867,0.20987017,0.122369766,0.29224345,2 +../Results/M0/images/12.png,0.30636993,0.23324025,0.46038982,0.24881713,0.1164028,0.31742933,2 +../Results/M0/images/13.png,0.3883742,0.1631384,0.44848734,0.32804036,0.068756245,0.36119783,2 +../Results/M0/images/14.png,0.33349174,0.3422713,0.3242369,0.18057151,0.15632431,0.2586949,1 +../Results/M0/images/15.png,0.41224623,0.29630646,0.29144734,0.2676859,0.16599588,0.30372384,0 +../Results/M0/images/16.png,0.20559113,0.21857733,0.57583153,0.1587709,0.06806885,0.21536787,2 +../Results/M0/images/17.png,0.26884133,0.25325018,0.47790846,0.19048718,0.08751682,0.2563862,2 +../Results/M0/images/18.png,0.21180235,0.186458,0.60173965,0.22845848,0.11598255,0.30478215,2 +../Results/M0/images/19.png,0.19250861,0.27735704,0.5301343,0.13503505,0.12023676,0.24182229,2 +../Results/M0/images/2.png,0.111913964,0.14639756,0.74168843,0.11043456,0.097838014,0{"code":"internal","msg":"git-diff-tree: context deadline exceeded","meta":{"cause":"*fmt.wrapError"}}