Merge branch 'main' into temp_contribution

Author: giospada

.devcontainer/Dockerfile

@@ -1,13 +1,14 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.9-buster
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.13
 USER vscode
 RUN curl -s "https://get.sdkman.io" | bash
-SHELL ["/bin/bash", "-c"]
-RUN source "/home/vscode/.sdkman/bin/sdkman-init.sh" && sdk install java 20.0.2-graalce
-RUN mkdir -p ~/lib && cd ~/lib && curl -L -O http://www.antlr.org/download/antlr-4.13.1-complete.jar
-ENV ANTLR_JAR="~/lib/antlr-4.13.1-complete.jar"
-RUN cd ~ && curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip && \
-    unzip protoc-25.1-linux-x86_64.zip -d ~/.local && \
-    rm protoc-25.1-linux-x86_64.zip
-RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.50.0/buf-$(uname -s)-$(uname -m)" -o ~/.local/bin/buf && chmod +x ~/.local/bin/buf
-RUN curl -LsSf https://astral.sh/uv/0.7.11/install.sh | sh
+RUN bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && \
+    sdk install java 25-graalce"
+RUN mkdir -p ~/lib && cd ~/lib && curl -L -O http://www.antlr.org/download/antlr-4.13.2-complete.jar
+ENV ANTLR_JAR="~/lib/antlr-4.13.2-complete.jar"
+# protoc 29.5 is the last version with protobuf python v5 which is compatible with protoletariat v3
+RUN cd ~ && curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.5/protoc-29.5-linux-x86_64.zip && \
+    unzip protoc-29.5-linux-x86_64.zip -d ~/.local && \
+    rm protoc-29.5-linux-x86_64.zip
+RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.58.0/buf-$(uname -s)-$(uname -m)" -o ~/.local/bin/buf && chmod +x ~/.local/bin/buf
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 USER root

.devcontainer/devcontainer.json

@@ -14,10 +14,21 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	// "postCreateCommand": "poetry install"
+	"postCreateCommand": "uv venv --clear && uv sync --extra test --extra gen_proto",
 
 	// Configure tool-specific properties.
-	// "customizations": {},
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"python.testing.pytestArgs": [
+					"tests"
+				],
+				"python.testing.unittestEnabled": false,
+				"python.testing.pytestEnabled": true
+			},
+			"extensions": []
+		}
+	}
 
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
 	// "remoteUser": "root"

.github/workflows/codegen-check.yml

@@ -0,0 +1,44 @@
+name: Code Generation Check
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  codegen-check:
+    name: Verify Code Generation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          submodules: recursive
+
+      - name: Run code generation in devcontainer
+        uses: devcontainers/[email protected]
+        with:
+          runCmd: |
+            # Ensure dependencies are installed
+            uv sync --extra test --extra gen_proto
+            # Run all code generation steps
+            make antlr
+            ./gen_proto.sh
+            make codegen-extensions
+
+      - name: Check for uncommitted changes
+        run: |
+          # Check for diffs, ignoring timestamp lines
+          if ! git diff --quiet --exit-code src/substrait/gen/; then
+            echo "Code generation produced changes. Generated code is out of sync!"
+            echo ""
+            git diff src/substrait/gen/
+            echo ""
+            echo "To fix this, run:"
+            echo "  make antlr"
+            echo "  ./gen_proto.sh"
+            echo "  make codegen-extensions"
+            echo "Then commit the changes."
+            exit 1
+          fi

.github/workflows/example.yml

@@ -0,0 +1,36 @@
+name: Run examples
+
+on:
+  pull_request:
+  push:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  example:
+    name: Run ${{ matrix.example }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        example:
+          - builder_example.py
+          - duckdb_example.py
+          - adbc_example.py
+          - pyarrow_example.py
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          submodules: recursive
+      - name: Install uv with python
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.10"
+      - name: Install package dependencies
+        run: |
+          uv sync --frozen --extra extensions
+      - name: Run ${{ matrix.example }}
+        run: |
+          uv run examples/${{ matrix.example }}

.github/workflows/pr_title.yml

@@ -8,7 +8,7 @@ jobs:
     name: PR title / description conforms to semantic-release
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v6
         with:
           node-version: "18"
       - run: npm install @commitlint/config-conventional

.github/workflows/release.yml

@@ -29,7 +29,7 @@ jobs:
         run: |
           python -m build
       - name: Upload package
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: dist
           path: dist/
@@ -41,7 +41,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Download artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: dist
           path: dist/
@@ -57,7 +57,7 @@ jobs:
     needs: release
     steps:
       - name: Download artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: dist
           path: dist/

.github/workflows/ruff.yml

@@ -15,15 +15,15 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ["3.9"]
+        python: ["3.10"]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
         with:
           submodules: recursive
       - name: Install uv with python
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ matrix.python }}
       - name: Run ruff linter

.github/workflows/test.yml

@@ -15,21 +15,20 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python: ["3.10", "3.11", "3.12", "3.13"]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
         with:
           submodules: recursive
-      - name: Set up Python
-        uses: actions/setup-python@v6
+      - name: Install uv with python
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ matrix.python }}
       - name: Install package and test dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install ".[test]"
+          uv sync --frozen --extra test
       - name: Run tests
         run: |
-          python -m pytest tests
+          uv run pytest

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "third_party/substrait"]
 	path = third_party/substrait
 	url = https://github.com/substrait-io/substrait
-[submodule "third_party/substrait-cpp"]
-	path = third_party/substrait-cpp
-	url = https://github.com/substrait-io/substrait-cpp

CONTRIBUTING.md

@@ -6,11 +6,10 @@ git clone --recursive https://github.com/<your-fork>/substrait-python.git
 cd substrait-python
 ```
 
-## Conda env
-Create a conda environment with developer dependencies.
+## Development environment
+Activate environment with uv.
 ```
-conda env create -f environment.yml
-conda activate substrait-python-env
+uv sync --extra test
 ```
 
 ## Update the substrait submodule locally
@@ -21,43 +20,45 @@ git submodule update --init --recursive
 ```
 
 
-# Upgrade the substrait protocol definition
+# Code generation
 
-## a) Use the upgrade script
+## Protobuf stubs
 
 Run the upgrade script to upgrade the submodule and regenerate the protobuf stubs.
 
 ```
-./update_proto.sh <version>
+uv sync --extra gen_proto
+uv run ./update_proto.sh <version>
 ```
 
-## b) Manual upgrade
+## Antlr grammar
 
-### Upgrade the Substrait submodule
+Substrait uses antlr grammar to derive output types of extension functions. Make sure java is installed and ANTLR_JAR environment variable is set. Take a look at .devcontainer/Dockerfile for example setup.
 
 ```
-cd third_party/substrait
-git checkout <version>
-cd -
-git commit . -m "Use submodule <version>"
+make antlr
 ```
 
-### Generate protocol buffers
-Generate the protobuf files manually. Requires protobuf `v3.20.1`.
+## Extensions stubs
+
+Substrait uses jsonschema to describe the data model for extension files.
+
 ```
-./gen_proto.sh
+make codegen-extensions
 ```
 
+# Lint & Format
+
+Run the following make commands to lint and format with ruff.
 
-# Build
-## Python package
-Editable installation.
 ```
-pip install -e .
+make lint
+make format
 ```
 
 # Test
 Run tests in the project's root dir.
 ```
-pytest
+uv sync --extra test
+uv run pytest
 ```