Training-Wiki/docker-compose.yml at main · PascalCTF/Training-Wiki · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
x-scraper-common: &scraper-common
  image: typesense/docsearch-scraper:latest
  user: root
  shm_size: '2gb'
  depends_on:
    - typesense
    - wiki
  environment:
    - TYPESENSE_API_KEY=${TYPESENSE_ADMIN_API_KEY:-xyz}
    - TYPESENSE_HOST=typesense
    - TYPESENSE_PORT=8108
    - TYPESENSE_PROTOCOL=http
    - CONFIG=/tmp/docsearch/docsearch.config.json
  volumes:
    - ./docsearch.config.json:/tmp/docsearch/docsearch.config.json:ro

services:
  typesense:
    image: typesense/typesense:30.2
    container_name: typesense
    ports:
      - "127.0.0.1:8108:8108"
    volumes:
      - typesense-data:/data
    command: >-
      --data-dir=/data
      --api-key=${TYPESENSE_ADMIN_API_KEY:-xyz}
      --enable-cors

  typesense-init:
    image: curlimages/curl:8.8.0
    container_name: typesense-init
    profiles: ["init"]
    depends_on:
      - typesense
    entrypoint: ["/bin/sh", "-c"]
    environment:
      - TYPESENSE_ADMIN_API_KEY=${TYPESENSE_ADMIN_API_KEY:-xyz}
      - TYPESENSE_SEARCH_API_KEY=${TYPESENSE_SEARCH_API_KEY:-xyz}
      - TYPESENSE_COLLECTION_NAME=${TYPESENSE_COLLECTION_NAME:-training-wiki}
    command:
      - >-
        until curl -sf http://typesense:8108/health >/dev/null; do sleep 1; done;
        curl -sf -X POST http://typesense:8108/keys
        -H "X-TYPESENSE-API-KEY: $$TYPESENSE_ADMIN_API_KEY"
        -H "Content-Type: application/json"
        -d "{\"description\":\"training-wiki search key\",\"actions\":[\"documents:search\"],\"collections\":[\"$$TYPESENSE_COLLECTION_NAME.*\"],\"value\":\"$$TYPESENSE_SEARCH_API_KEY\"}"
        >/dev/null || true

  wiki:
    build:
      context: .
      args:
        TYPESENSE_COLLECTION_NAME: ${TYPESENSE_COLLECTION_NAME:-training-wiki}
        TYPESENSE_HOST: ${TYPESENSE_HOST:-localhost}
        TYPESENSE_PORT: ${TYPESENSE_PORT:-8108}
        TYPESENSE_PROTOCOL: ${TYPESENSE_PROTOCOL:-http}
        TYPESENSE_SEARCH_API_KEY: ${TYPESENSE_SEARCH_API_KEY:-xyz}
    container_name: wiki
    depends_on:
      - typesense
    ports:
      - "127.0.0.1:1337:3000"
      - "6007:22"
    volumes:
      - scraper-trigger:/trigger
    environment:
      - ROOT_PASSWORD=${ROOT_PASSWORD}
      - TYPESENSE_COLLECTION_NAME=${TYPESENSE_COLLECTION_NAME:-training-wiki}
      - TYPESENSE_HOST=${TYPESENSE_HOST:-localhost}
      - TYPESENSE_PORT=${TYPESENSE_PORT:-8108}
      - TYPESENSE_PROTOCOL=${TYPESENSE_PROTOCOL:-http}
      - TYPESENSE_SEARCH_API_KEY=${TYPESENSE_SEARCH_API_KEY:-xyz}

  scraper:
    <<: *scraper-common
    container_name: scraper
    profiles: ["scraper"]

  scraper-watcher:
    <<: *scraper-common
    container_name: scraper-watcher
    restart: unless-stopped
    volumes:
      - ./docsearch.config.json:/tmp/docsearch/docsearch.config.json:ro
      - scraper-trigger:/trigger
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >-
        set -eu;
        trigger="/trigger/run";
        echo "[scraper-watcher] waiting for trigger...";
        while true; do
          if [ -f "$$trigger" ]; then
            rm -f "$$trigger";
            echo "[scraper-watcher] trigger detected, scraping...";
            pipenv run python -m src.index || echo "[scraper-watcher] scrape failed";
            echo "[scraper-watcher] done, waiting for next trigger...";
          fi;
          sleep 5;
        done

volumes:
  typesense-data:
  scraper-trigger: