diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 8de3c9e4..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "uv" - directory: "/" - schedule: - interval: "daily" - # Only use this to bump our libraries - allow: - - dependency-name: "unstructured[all-docs]" - - - package-ecosystem: "github-actions" - # NOTE(robinson) - Workflow files stored in the - # default location of `.github/workflows` - directory: "/" - schedule: - interval: "weekly" diff --git a/.github/workflows/bump_libraries.yaml b/.github/workflows/bump_libraries.yaml deleted file mode 100644 index 555a052c..00000000 --- a/.github/workflows/bump_libraries.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: Dependabot - Bump libs and cut release - -on: - pull_request: - types: - - opened - - reopened - paths: - - 'uv.lock' - - 'pyproject.toml' - -jobs: - bump-changelog: - runs-on: opensource-linux-8core - if: ${{ github.actor == 'dependabot[bot]' }} - permissions: - contents: write - steps: - - uses: actions/checkout@v5 - - name: Read Python version from .python-version - run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV - - name: Install uv - uses: astral-sh/setup-uv@v7 - - name: Set up Python ${{ env.PYTHON_VERSION }} - run: uv python install ${{ env.PYTHON_VERSION }} - - name: Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@v2 - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - name: Create release version - run: | - uv lock --upgrade - package=${{ steps.metadata.outputs.dependency-names }} - # Strip any [extras] from name - package=${package%\[*} - changelog_message="Bump $package to ${{ steps.metadata.outputs.new-version }}" - ./scripts/version-increment.sh "$changelog_message" - make version-sync - - uses: stefanzweifel/git-auto-commit-action@v6 - with: - commit_message: "Bump libraries and release" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index cd3a5b54..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: CI - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - PIPELINE_FAMILY: "general" - -jobs: - lint: - runs-on: opensource-linux-8core - steps: - - uses: actions/checkout@v5 - - name: Read Python version from .python-version - run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - cache-dependency-glob: "uv.lock" - - name: Set up Python ${{ env.PYTHON_VERSION }} - run: uv python install ${{ env.PYTHON_VERSION }} - - name: Install lint dependencies - run: uv sync --only-group lint --frozen - - name: Lint - run: make check - - shellcheck: - runs-on: opensource-linux-8core - steps: - - uses: actions/checkout@v5 - - name: ShellCheck - uses: ludeeus/action-shellcheck@master - - test: - runs-on: opensource-linux-8core - needs: lint - steps: - - uses: actions/checkout@v5 - - name: Read Python version from .python-version - run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - cache-dependency-glob: "uv.lock" - - name: Set up Python ${{ env.PYTHON_VERSION }} - run: uv python install ${{ env.PYTHON_VERSION }} - - name: Install dependencies and run core tests - run: | - sudo apt-get update && sudo apt-get install --yes poppler-utils libreoffice - uv sync --group test --frozen - make install-pandoc - make install-nltk-models - sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5 - sudo apt-get install -y tesseract-ocr tesseract-ocr-kor - tesseract --version - make test - make check-coverage - - changelog: - runs-on: opensource-linux-8core - steps: - - uses: actions/checkout@v5 - - if: github.ref != 'refs/heads/main' - uses: dorny/paths-filter@v3 - id: changes - with: - filters: | - src: - - 'doc_recipe/**' - - 'recipe-notebooks/**' - - - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main' - uses: dangoslen/changelog-enforcer@v3 - - # TODO - figure out best practice for caching docker images - test_dockerfile: - runs-on: opensource-linux-8core - needs: lint - steps: - - uses: actions/checkout@v5 - - name: Read Python version from .python-version - run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - cache-dependency-glob: "uv.lock" - - name: Set up Python ${{ env.PYTHON_VERSION }} - run: uv python install ${{ env.PYTHON_VERSION }} - - name: Free up disk space - run: | - # Clear some space (https://github.com/actions/runner-images/issues/2840) - echo "Disk usage before cleanup:" - df -h - - # Remove unnecessary pre-installed software - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /usr/local/share/boost - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo rm -rf /usr/local/.ghcup - sudo rm -rf /usr/share/swift - - # Clean up docker to ensure we start fresh - docker system prune -af --volumes - - echo "Disk usage after cleanup:" - df -h - - name: Test Dockerfile - run: | - uv sync --group test --frozen - make docker-build - make docker-test diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index 3b46a4fd..00000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: opensource-linux-8core - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - fetch-depth: 1 - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@beta - with: - anthropic_api_key: ${{ secrets.GH_ANTHROPIC_API_KEY }} - allowed_tools: "Bash(git:*),View,GlobTool,GrepTool,BatchTool" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 4c3786d6..00000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Build And Push Docker Image - -on: - push: - branches: - - main - workflow_dispatch: - -env: - DOCKER_REPOSITORY: quay.io/unstructured-io/unstructured-api - DOCKER_BUILD_REPOSITORY: quay.io/unstructured-io/build-unstructured-api - PACKAGE: "unstructured-api" - PIPELINE_FAMILY: "general" - -jobs: - set-short-sha: - runs-on: opensource-linux-8core - outputs: - short_sha: ${{ steps.set_short_sha.outputs.short_sha }} - steps: - - name: Set Short SHA - id: set_short_sha - run: echo "short_sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT - build-images: - strategy: - matrix: - arch: ["arm64", "amd64"] - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'opensource-linux-8core' }} - needs: set-short-sha - env: - SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }} - DOCKER_PLATFORM: linux/${{ matrix.arch }} - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - driver: docker - - name: Checkout code - uses: actions/checkout@v5 - - name: Read Python version from .python-version - run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV - - name: Login to Quay.io - uses: docker/login-action@v4 - with: - registry: quay.io - username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }} - password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }} - - name: Free up disk space - run: | - # Clear some space (https://github.com/actions/runner-images/issues/2840) - echo "Disk usage before cleanup:" - df -h - - # Remove unnecessary pre-installed software - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /usr/local/share/boost - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo rm -rf /usr/local/.ghcup - sudo rm -rf /usr/share/swift - - # Clean up docker to ensure we start fresh - docker system prune -af --volumes - - echo "Disk usage after cleanup:" - df -h - - name: Build image - run: | - DOCKER_BUILDKIT=1 docker buildx build --load -f Dockerfile \ - --platform=$DOCKER_PLATFORM \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - --build-arg PIPELINE_PACKAGE=${{ env.PIPELINE_FAMILY }} \ - --provenance=false \ - --progress plain \ - --cache-from $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }} \ - -t $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA . - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - cache-dependency-glob: "uv.lock" - - name: Set up Python ${{ env.PYTHON_VERSION }} - run: uv python install ${{ env.PYTHON_VERSION }} - - name: Install test dependencies - run: uv sync --group test --frozen - - name: Test image - run: | - export DOCKER_IMAGE="$DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA" - SKIP_INFERENCE_TESTS=true make docker-test - - name: Push image - run: | - # write to the build repository to cache for the publish-images job - docker push $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA - publish-images: - runs-on: opensource-linux-8core - needs: [set-short-sha, build-images] - env: - SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }} - steps: - - name: Checkout code - uses: actions/checkout@v5 - - name: Set SHORT_SHA - run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - name: Login to Quay.io - uses: docker/login-action@v4 - with: - registry: quay.io - username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }} - password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }} - - name: Pull AMD image - run: | - docker pull $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA - - name: Pull ARM image - run: | - docker pull $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA - - name: Push AMD and ARM tags - run: | - # these are used to construct the final manifest but also cache-from in subsequent runs - docker tag $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64 - docker push $DOCKER_BUILD_REPOSITORY:amd64 - docker tag $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:arm64 - docker push $DOCKER_BUILD_REPOSITORY:arm64 - - name: Push multiarch manifest - run: | - VERSION=$(grep -oP '(?<=__version__ = ")[^"]+' prepline_general/api/__version__.py) - docker buildx imagetools create \ - -t ${DOCKER_REPOSITORY}:latest \ - -t ${DOCKER_REPOSITORY}:$SHORT_SHA \ - -t ${DOCKER_REPOSITORY}:$VERSION \ - $DOCKER_BUILD_REPOSITORY:amd64 \ - $DOCKER_BUILD_REPOSITORY:arm64 diff --git a/.github/workflows/schoolai-build-publish.yml b/.github/workflows/schoolai-build-publish.yml index 4601136c..509d1a29 100644 --- a/.github/workflows/schoolai-build-publish.yml +++ b/.github/workflows/schoolai-build-publish.yml @@ -1,34 +1,39 @@ -name: SchoolAI Build and Publish +name: unstructured-api-staging-build -# Builds the SchoolAI fork image and publishes it to our Artifact Registry so -# the webapp (apps/web RAG pipeline) can run a build. +# Builds the SchoolAI fork image and publishes to GCP Artifact Registry. +# Pattern matches web-app service workflows (e.g. lms-mcp-server-staging-deploy, +# sentinel-staging-deploy): push to main triggers sai-docker-build on SchoolAI runners. # -# Mirrors the webapp build pattern (see web-app: -# .github/workflows/webapp-staging-deploy.yml -> build_webapp) by reusing the -# shared SchoolAI/github-actions-workflows sai-docker-build.yaml workflow. +# Deploy is handled by web-app (webapp-deploy-argocd deploy_unstructured_api), which +# references us-central1-docker.pkg.dev/schoolai-global/docker/unstructured-api:latest-amd64 +# in ops/unstructured-api/values-global.yaml. A new image is picked up on the next deploy. +# +# Upstream Unstructured-IO workflows were removed from this fork. +# Only this build workflow runs in SchoolAI CI. on: - workflow_call: - # Keep workflow_dispatch so operators can trigger a manual staging deploy - # even though `staging-deploy.yml` is now the primary entrypoint on main. + push: + branches: + - main workflow_dispatch: +permissions: + contents: read + jobs: build_unstructured_api: uses: SchoolAI/github-actions-workflows/.github/workflows/sai-docker-build.yaml@main with: - # amd64 only — matches the cluster nodeSelector (kubernetes.io/arch: amd64) - # and how the other SchoolAI services build. build_amd_enabled: true build_arm_enabled: false docker_build_amd_runs_on: sai-7cpu-26gb-amd-perf - # Produces images at: - # us-central1-docker.pkg.dev/schoolai-global/docker/unstructured-api: docker_repo: us-central1-docker.pkg.dev/schoolai-global/docker docker_image_name: unstructured-api dockerfile: ./Dockerfile docker_build_args: | PIPELINE_PACKAGE=general push_image: true + push_cache: true secrets: GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_STAGING }} + SLACK_WEBHOOK: ${{ secrets.SLACK_STAGING_DEPLOYMENTS }} diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml deleted file mode 100644 index c5549b87..00000000 --- a/.github/workflows/version-bump.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Version Bump - -on: - pull_request: - branches: [main] - types: [opened, synchronize, reopened] - -permissions: - contents: write - pull-requests: read - -jobs: - version-bump: - if: github.event.pull_request.user.login == 'utic-renovate[bot]' - uses: Unstructured-IO/infra/.github/workflows/version-bump.yml@main - with: - component-paths: '["."]' - default-bump: patch - update-changelog: true - update-lockfile: true - renovate-app-id: ${{ vars.RENOVATE_APP_ID }} - secrets: - token: ${{ secrets.GITHUB_TOKEN }} - private-pypi-url: ${{ secrets.PRIVATE_PYPI_INDEX_URL }} - renovate-app-private-key: ${{ secrets.RENOVATE_APP_PRIVATE_KEY }}