Skip to content

Commit 55aca86

Browse files
TrinaKatTrinaKat
authored
Merge pull request #134 from ucladevx/kat-python3
Moved Backend to Python 3 and Refactored Time-Intensive Dependencies
2 parents e02f2be + 01201af commit 55aca86

File tree

8 files changed

+75
-187
lines changed

8 files changed

+75
-187
lines changed

‎Makefile‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22
include .env
33

44
ECR_REPO=698514710897.dkr.ecr.us-west-1.amazonaws.com
5+
BASE_NAME=mappening/base
56
APP_NAME=mappening/backend
67
DEV_NAME=mappening/dev
78
PROD_DOCKERFILE=./src/Dockerfile
8-
DEV_DOCKERFILE=./src/Dev-Dockerfile
9+
BASE_DOCKERFILE=./src/Base-Dockerfile
10+
DEV_DOCKER_COMPOSE=./docker-compose-dev.yml
911

1012
########################## AWS / PRODUCTION ##########################
1113

@@ -35,6 +37,10 @@ dora: ecr-login build-dora
3537

3638
################## LOCAL DEVELOPMENT (Backend Only) ##################
3739

40+
# Build backend image
41+
build-base:
42+
docker build ./src -t $(BASE_NAME) -f $(BASE_DOCKERFILE)
43+
3844
# Build backend image
3945
build-dev:
4046
docker build ./src -t $(APP_NAME) -f $(DEV_DOCKERFILE)
@@ -56,7 +62,10 @@ stop:
5662

5763
################## IN PROGRESS: POSTGRES ##################
5864

59-
up:
65+
up-dev:
66+
docker-compose -f $(DEV_DOCKER_COMPOSE) up --build
67+
68+
up-prod:
6069
docker-compose up --build
6170

6271
# Stops the stack. Can also Ctrl+C in the same terminal window stack was run.

‎docker-compose-dev.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
version: "3"
2+
services:
3+
4+
postgres:
5+
build: ./database
6+
restart: always
7+
ports:
8+
- ${POSTGRES_PORT}:${POSTGRES_PORT}
9+
environment:
10+
- POSTGRES_USER=${POSTGRES_USER}
11+
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
12+
volumes:
13+
- ./database/postgres:/var/lib/postgresql/data
14+
15+
api:
16+
build: ./src
17+
ports:
18+
- ${API_SERVER_PORT}:${API_SERVER_PORT}
19+
command: bash -c 'while !</dev/tcp/postgres/5432; do sleep 2; done; python3 app.py'
20+
environment:
21+
- POSTGRES_URL=${POSTGRES_URL}
22+
depends_on:
23+
- postgres

‎docker-compose.yml‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,10 @@
11
version: "3"
22
services:
33

4-
postgres:
5-
build: ./database
6-
restart: always
7-
ports:
8-
- ${POSTGRES_PORT}:${POSTGRES_PORT}
9-
environment:
10-
- POSTGRES_USER=${POSTGRES_USER}
11-
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
12-
volumes:
13-
- ./database/postgres:/var/lib/postgresql/data
14-
154
api:
165
build: ./src
176
ports:
187
- ${API_SERVER_PORT}:${API_SERVER_PORT}
19-
command: bash -c 'while !</dev/tcp/postgres/5432; do sleep 2; done; python app.py'
8+
command: python3 app.py -p
209
environment:
21-
- POSTGRES_URL=${POSTGRES_URL}
22-
depends_on:
23-
- postgres
10+
- POSTGRES_URL=${AWS_PG_URL}

‎src/Base-Dockerfile‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Takes around 15-20 minutes to run the first time (depending on your computer)
2+
FROM python:3.7-alpine3.8
3+
4+
LABEL maintainer="[email protected]"
5+
6+
ENV ALPINE_VERSION=3.8 \
7+
PACKAGES="bash ca-certificates python3 py-setuptools curl"
8+
9+
# Set the PYTHON UNICODE
10+
ENV PYTHONIOENCODING UTF-8
11+
12+
# Install geos-dev and some other packages for shapely
13+
RUN apk update && apk upgrade && apk add --no-cache --update build-base git
14+
RUN echo "http://mirror.leaseweb.com/alpine/edge/testing/" >> /etc/apk/repositories
15+
RUN apk add --no-cache geos-dev
16+
17+
# Install needed packages
18+
# https://github.com/jfloff/alpine-python/issues/24?fbclid=lwAR08KeopkO3ET6hz8iZ6bV0hEljjpHZaKR2bXMZAUjJsKnauzDLflzeTdpTY
19+
RUN set -ex ;\
20+
echo "@community http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_VERSION/community" >> /etc/apk/repositories ;\
21+
apk add --no-cache --update \
22+
openblas-dev@community \
23+
expat@community;
24+
25+
# Install pip packages to:
26+
# 1) deal with order dependencies
27+
# 2) save us a ton of time for future requirements.txt modifications
28+
RUN pip3 install git+https://github.com/mitsuhiko/flask-oauth
29+
RUN pip3 install numpy scipy gensim
30+
RUN pip3 install Cython
31+
RUN pip3 install scikit-learn
32+
RUN pip3 install pandas
33+
34+
# https://gist.github.com/orenitamar/f29fb15db3b0d13178c1c4dd611adce2
35+
RUN apk add --update-cache bash gcc gfortran python python-dev py-pip wget freetype-dev libpng-dev expat postgresql-dev musl-dev

‎src/Dev-Dockerfile‎

Lines changed: 0 additions & 86 deletions
This file was deleted.

‎src/Dockerfile‎

Lines changed: 3 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,13 @@
11
# Takes around 15-20 minutes to run the first time (depending on your computer)
2-
FROM alpine:3.6
3-
4-
ENV ALPINE_VERSION=3.6 \
5-
PACKAGES="bash ca-certificates python2 py-setuptools curl"
6-
7-
# Set the PYTHON STUPID UNICODE EOFUBDHSKJOIWEJNOIEFNODKL
8-
ENV PYTHONIOENCODING UTF-8
9-
10-
# Replacing default repositories with edge ones
11-
RUN echo \
12-
# && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" > /etc/apk/repositories \
13-
&& echo "http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_VERSION/community" >> /etc/apk/repositories
14-
# && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories
15-
16-
# Add the packages, with a CDN-breakage fallback if needed
17-
RUN echo \
18-
&& apk add --no-cache $PACKAGES || \
19-
(sed -i -e 's/dl-cdn/dl-4/g' /etc/apk/repositories && apk add --no-cache $PACKAGES)
20-
21-
# turn back the clock - hacky lmao
22-
RUN echo \
23-
# && echo "http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_VERSION/main/" > /etc/apk/repositories
24-
# && echo "@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
25-
# && echo "@edge-community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
26-
# && echo "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \
27-
28-
# make some useful symlinks that are expected to exist
29-
RUN echo \
30-
&& if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python2.7 /usr/bin/python; fi \
31-
&& if [[ ! -e /usr/bin/python-config ]]; then ln -sf /usr/bin/python2.7-config /usr/bin/python-config; fi \
32-
&& if [[ ! -e /usr/bin/easy_install ]]; then ln -sf /usr/bin/easy_install-2.7 /usr/bin/easy_install; fi
33-
34-
# Install and upgrade Pip
35-
RUN echo \
36-
&& easy_install pip \
37-
&& pip install --upgrade pip \
38-
&& if [[ ! -e /usr/bin/pip ]]; then ln -sf /usr/bin/pip2.7 /usr/bin/pip; fi
39-
40-
# Install geos-dev and some other packages for shapely
41-
RUN apk update && apk upgrade && apk add --no-cache --update build-base
42-
43-
# Add GEOS for shapely
44-
RUN echo "http://mirror.leaseweb.com/alpine/edge/testing/" >> /etc/apk/repositories
45-
RUN apk add --no-cache geos-dev
46-
47-
# Install needed packages
48-
# https://github.com/jfloff/alpine-python/issues/24?fbclid=lwAR08KeopkO3ET6hz8iZ6bV0hEljjpHZaKR2bXMZAUjJsKnauzDLflzeTdpTY
49-
RUN set -ex ;\
50-
echo "@community http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_VERSION/community" >> /etc/apk/repositories ;\
51-
apk add --no-cache --update \
52-
openblas-dev@community \
53-
expat@community \
54-
;\
55-
rm /var/cache/apk/*
56-
57-
# Install numpy and scipy for ml categories
58-
# Takes 5 years to run... building this part as the base image
59-
# It is cached but just in case we're adding this image
60-
# https://gist.github.com/orenitamar/f29fb15db3b0d13178c1c4dd611adce2
61-
# Dmaa split these into separate lines for debugging
62-
RUN apk add --update-cache gcc
63-
RUN apk add --update-cache gfortran
64-
RUN apk add --update-cache python
65-
RUN apk add --update-cache python-dev
66-
RUN apk add --update-cache py-pip
67-
RUN apk add --update-cache build-base
68-
RUN apk add --update-cache wget
69-
RUN apk add --update-cache freetype-dev
70-
RUN apk add --update-cache libpng-dev
71-
RUN apk add --update-cache expat
72-
RUN apk add --update-cache postgresql-dev
73-
RUN apk add --update-cache musl-dev
74-
RUN pip install --upgrade pip
75-
RUN pip install numpy scipy gensim
2+
FROM mappening/base
763

774
# Install the rest of the packages
785
COPY requirements.txt /tmp
79-
RUN cd /tmp && pip install -r requirements.txt
6+
RUN cd /tmp && pip3 install -r requirements.txt
807

818
WORKDIR /app
829
COPY . .
8310
RUN ls -l
8411

8512
EXPOSE 5000
86-
CMD ["python", "app.py", "-p"]
13+
CMD ["python3", "app.py"]

‎src/mappening/utils/scheduler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
# "return a StringIO-like stream for reading and writing"
1010
# basically allows stdoutput to be saved into a stream
11-
from cStringIO import StringIO
11+
from io import StringIO
1212
import sys, traceback
1313

1414
# (hour, minute) pairs for when to refresh events every day (using 24 hour clock), IN LA TIME

‎src/requirements.txt‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
flask
22
flask_cors
3-
flask-oauth
43
flask_compress
54
flask_login
65
pymongo
@@ -17,9 +16,3 @@ fuzzywuzzy
1716
unidecode
1817
psycopg2
1918
Flask-SQLAlchemy
20-
21-
# Machine Learning
22-
scikit-learn
23-
numpy
24-
scipy
25-
pandas

0 commit comments

Comments
 (0)