merge from master

.dockerignore

@@ -8,4 +8,11 @@
 .git
 .env
 
-pytest_log.txt
+pytest_log.txt
+
+**/eval/**/result
+
+import/simple/sample/
+**/.data
+**/testdata
+**/test_data

server/.gcloudignore → .gcloudignore

@@ -12,17 +12,21 @@
 # below:
 .git
 .gitignore
+#!include:.gitignore
+#!include:.dockerignore
+!mcf_parser.py
 
 # Python pycache:
 **/__pycache__/**
-.pytest_cache
+.pytest_cache/
 
 # Ignored by the build system
 /setup.cfg
 
-**/node_modules/**
+static/node_modules/
 
-.env/
+server/dist/
 
-# Ignore tests
-tests/
+**/.venv/
+**/.venv_website/
+**/.venv_nl/

.gemini/config.yaml

@@ -0,0 +1,13 @@
+# Review settings for Gemini Code Assist in GitHub
+# Documentation: https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github#add-configuration-files
+have_fun: false
+code_review:
+  disable: false
+  comment_severity_threshold: MEDIUM
+  max_review_comments: -1
+  pull_request_opened:
+    help: false
+    summary: true
+    code_review: true
+    include_drafts: true
+ignore_patterns: ["import/**", "mixer/**"]

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# Specific rule for a particular file:
+# This line should come AFTER the general rule.
+# The last matching pattern takes precedence.
+server/config/feature_flag_config/production.json @datacommonsorg/frontend

.github/workflows/codeql-analysis.yml

@@ -20,6 +20,11 @@ on:
   schedule:
     - cron: "30 7 * * 3"
 
+permissions:
+  contents: read
+  security-events: write
+  pull-requests: read
+
 jobs:
   analyze:
     name: Analyze
@@ -35,11 +40,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +55,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -64,4 +69,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/release-branch-checks.yml

@@ -0,0 +1,54 @@
+name: Release branch checks
+
+on:
+  pull_request:
+    branches: ["customdc_stable"]
+  # Required for merge queue to work: https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions
+  merge_group:
+    branches: ["customdc_stable"]
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  verify_all_commits_are_already_in_master:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch all history for accurate comparison
+          fetch-depth: 0
+          # Check out the PR branch
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Verify that all commits are already in the master branch
+        run: |
+          git remote add dc https://github.com/datacommonsorg/website.git
+          git fetch dc
+          MASTER_BRANCH="dc/master"
+
+          # Get the list of commits in the source branch that are not in the master branch.
+          # Exclude merge commits only if this is the final run in the merge queue.
+          # This way the only merge commits that end up in the final commit history
+          # are the ones added by GitHub when merging PRs.
+          if [[ ${{ github.event_name }}  == 'merge_group' ]]; then
+            MISSING_COMMITS=$(git log --pretty="%H - %s" --no-merges $MASTER_BRANCH..HEAD --)
+          else
+            MISSING_COMMITS=$(git log --pretty="%H - %s" $MASTER_BRANCH..HEAD --)
+          fi
+
+          if [[ -n "$MISSING_COMMITS" ]]; then
+            echo ""
+            echo "ERROR: The following commits are not present in $MASTER_BRANCH:"
+            echo ""
+            echo "$MISSING_COMMITS"
+            echo ""
+            echo "PRs to release branches should only contain commits that are already in master."
+            echo "To fix this PR, reset its branch locally to a commit at or behind https://github.com/datacommonsorg/website/commits/master/ and then force-push it."
+            echo "Note that a release branch PR should be based on master and not the previous version of the release branch, which contains merge commits."
+            exit 1
+          fi
+
+          echo "All commits are present in $MASTER_BRANCH"

.github/workflows/submodule-update.yml

@@ -0,0 +1,88 @@
+name: Update Submodules
+
+on:
+  schedule:
+    # Run every day at midnight EST (00:00 EST / 05:00 UTC)
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    # Allows manual triggering of the workflow
+
+jobs:
+  update-submodules:
+    if: github.repository == 'datacommonsorg/website'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # Needed to push changes
+      pull-requests: write # Needed to create pull requests
+      issues: write # Needed for label creation
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Use the PAT from your robot account for write permissions
+          token: ${{ secrets.SUBMOD_UPDATE_TOKEN }}
+          submodules: recursive # Initialize and update submodules
+          fetch-depth: 0 # Fetch all history for all branches and tags
+
+      - name: Configure Git
+        run: |
+          # Set the Git user to your robot account's details
+          git config user.name "datacommons-robot-author"
+          git config user.email "[email protected]"
+
+      - name: Update submodules to latest
+        id: update_submodules
+        run: |
+          # update all submodules
+          git submodule update --remote --merge
+
+          # Check if there are any changes to submodules
+          if [[ -z $(git status --porcelain) ]]; then
+            echo "No submodule updates found."
+            echo "submodules_updated=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "Submodules updated. Committing changes."
+            git add mixer import # Add only the submodules if they are the only changes expected
+            git commit -m "chore: Update submodules [skip ci]"
+            echo "submodules_updated=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Get current date
+        id: date
+        run: echo "today=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
+
+      - name: Create Pull Request
+        id: cpr
+        if: steps.update_submodules.outputs.submodules_updated == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.SUBMOD_UPDATE_TOKEN }}
+          commit-message: "chore: Update submodules [skip ci]"
+          title: "chore: Update submodules"
+          body: "This PR automatically updates the `mixer` and `import` submodules to their latest `master` branches."
+          # Use the date from the previous step
+          branch: "chore/update-submodules-${{ steps.date.outputs.today }}" # New branch for the PR
+          base: "master" # Target branch for the PR
+          labels: automated-pr, dependencies
+          add-paths: 'mixer,import' # Explicitly add paths to be included in the PR
+          delete-branch: true # Delete the branch after PR merge
+
+      - name: Auto-Merge Pull Request with PAT (Bypassing Reviews)
+        # This step runs only if a PR was successfully created in the previous step
+        if: steps.cpr.outputs.pull-request-number != ''
+        env:
+         GITHUB_TOKEN: ${{ secrets.SUBMOD_UPDATE_TOKEN }}
+        run: |
+         PR_NUMBER=${{ steps.cpr.outputs.pull-request-number }}
+
+         echo "Attempting to merge PR #$PR_NUMBER using PAT and --admin flag..."
+
+         # The 'gh' (GitHub CLI) is pre-installed on GitHub Actions runners.
+         # --admin: This flag is crucial. It tells 'gh' to bypass branch protection rules
+         #          (like required reviews or passing status checks).
+         # --squash: Specifies the merge method (you can also use --merge or --rebase).
+         # --delete-branch: Deletes the source branch of the PR after merging.
+         gh pr merge "$PR_NUMBER" --admin --squash --delete-branch
+
+         echo "Pull request #$PR_NUMBER merged successfully."

.gitignore

@@ -2,19 +2,29 @@
 __pycache__/
 .pytest_cache
 *.pyc
-.env/
-test_screenshots/
+.venv*/
+
+# Gemini
+.gemini/**
+.geminiignore
+!.gemini/
+!.gemini/config.yaml
+
 
 # Production static files
-server/dist/**
+**/dist/**
 !server/dist/.empty
 
 **/node_modules/**
 
 # Local env.
 *.swp
 .vscode
+.idea
 !.vscode/launch.json
+.env
+.env.list
+build/cdc/dev/.env-test
 
 # Mac
 .DS_Store
@@ -24,9 +34,11 @@ tmp/
 .air.toml
 
 # Kubernetes
-gke/config.yaml
 gke/endpoints.yaml
 gke/mci.yaml
+endpoints.yaml
+gke/*.crt
+gke/*.key
 
 mixer-grpc.*.pb
 deploy/overlays/**/endpoints.yaml
@@ -37,6 +49,8 @@ deploy/overlays/kustomization.yaml
 deploy/git/mixer_hash.txt
 deploy/git/website_hash.txt
 
+deploy/apigee/**/*.zip
+
 # Helm
 build/helm/**/*.tgz
 
@@ -49,4 +63,57 @@ build/helm/**/*.tgz
 *.tfstate.*
 pylist.json
 
-pytest_log.txt
+pytest_log.txt
+
+
+tools/sdg/sdg.textproto
+tools/sdg/subject_page_pb2.py
+
+*diff_report.html
+
+# Experimental - ignore symbolic links
+experimental/sdg-static/datacommons/datacommons.js
+experimental/sdg-static/datacommons/ranking.min.css
+experimental/sdg-static/datacommons/nl_interface.min.css
+
+# Custom DC data
+dc-data/
+custom_dc/env.list
+
+# Topic cache
+gen_ordered_list_for_topics.mcf
+mcf_parser.py
+
+# Test output files
+output/
+
+# wireit files
+.wireit
+
+# finetuned model
+ft_final_v20230717230459.all-MiniLM-L6-v2/
+
+# model eval output
+
+**/eval/**/result
+
+# CDC ignores
+custom_dc/sample/datacommons/**
+custom_dc/data/*.db
+
+.cache/
+
+**/*.egg-info
+*.egg-info
+
+# Custom DC dev and terraform ignores
+build/cdc/dev/.env
+*.tfvars
+deploy/terraform-custom-datacommons/modules/backend.tf
+
+# NL tools ignores
+tools/nl/nl_metadata/.env
+
+# Gemini files
+GEMINI.md
+.gemini/

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "mixer"]
 	path = mixer
 	url = https://github.com/datacommonsorg/mixer.git
+[submodule "import"]
+	path = import
+	url = https://github.com/datacommonsorg/import.git