T397332 use poetry

.github/workflows/build-and-push.yaml

@@ -10,13 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      - name: git fetch
-        run: |
-          git fetch
-      - name: git checkout
-        run: |
-          git checkout ${{ github.head_ref }}
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.head_repo.full_name }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/tests.yaml

@@ -9,7 +9,10 @@ jobs:
   tox:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.head_repo.full_name }}
 
       - name: build tox container
         run: |
@@ -21,6 +24,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.head_repo.full_name }}
       - name: Install dependencies
         run: npm ci
       - name: Run tests

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker-registry.wikimedia.org/python3-bookworm:latest
+FROM docker-registry.wikimedia.org/python3-trixie:latest
 
 # Create Quarry user, create /results folder owned by this user,
 # to be mounted as volume to be shared between web and runner in dev setup
@@ -8,13 +8,20 @@ RUN useradd -r -m quarry && \
 
 WORKDIR /app
 
-COPY requirements.txt /app
-# Install dependencies
-# TODO: Use a venv instead of --break-system-packages
-# TODO: Use newer pip. That requires newer celery, which in turn requires
-# newer versions of basically everything else.
-RUN pip install --break-system-packages --upgrade pip==24.0 wheel && \
-    pip install --break-system-packages -r requirements.txt
+# 1. Update pip, install Poetry, and set venv path
+RUN pip install --break-system-packages --ignore-installed --upgrade pip wheel && \
+    pip install --break-system-packages poetry
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+ENV PATH="/app/.venv/bin:$PATH"
+
+# 2. Copy dependency files
+COPY pyproject.toml poetry.lock /app/
+
+# 3. Install dependencies via Poetry (no --break-system-packages needed)
+RUN poetry install --no-root --only main --no-interaction
+
+# 4. Ensure quarry user can access the virtual environment
+RUN chown -R quarry:quarry /app/.venv
 
 # Copy app code
 USER quarry

README.md

@@ -60,6 +60,29 @@ If you had already run a dev environment (that is, ran `docker-compose up`) you
 the containers with the new dependencies by running `docker-compose build` before running
 `docker-compose up` again.
 
+### Direct Local Development (Non-Containerized) ###
+
+If you prefer to run the Flask application directly on your machine without Docker, you must first set up the Python environment using Poetry.
+
+**Install Poetry:**
+```bash
+pip install poetry
+```
+
+**Install Dependencies:** Run this command from the project root. It will create a local virtual environment and install all required packages.
+```bash
+poetry install
+```
+
+**Activate Shell and Run:** Run your application (e.g., for testing or running locally):
+```bash
+poetry shell
+flask run
+```
+
+**Note:** You may need to run local Redis and database services separately to avoid ConnectionErrors.
+**Note:** You may need to install some dependencies to be able to bulid the python wheels pulled by pip (like greenlet), for example `sudo dnf install python3-devel gcc-c++` on fedora systems.
+
 ## Useful commands ##
 
 To pre-compile nunjucks templates:
@@ -168,4 +191,4 @@ If ansible doesn't detect a change for quarry helm the following can be run:
 `helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml`
 
 For shell access, a debug pod can be created on a running node with something lie
-$ kubectl debug node/quarry-127a-g4ndvpkr5sro-node-0 -it --image debian:stable  
+$ kubectl debug node/quarry-127a-g4ndvpkr5sro-node-0 -it --image debian:stable

docker-compose.yml

@@ -6,10 +6,9 @@ services:
     build: .
     # To mimic a production runtime, uncomment:
     # entrypoint: ["gunicorn", "-w", "2", "--bind", "0.0.0.0:5000", "wsgi:application"]
-    entrypoint: ["python3", "wsgi.py"]
+    entrypoint: ["poetry", "run", "python3", "wsgi.py"]
     volumes:
-      - .:/app
-      - results:/results
+      - results:/results:rw,Z
     ports:
       - "5000:5000"
     depends_on:
@@ -20,9 +19,8 @@ services:
   worker:
     build: .
     volumes:
-      - .:/app
-      - results:/results
-    entrypoint: ["celery", "--app", "quarry.web.worker", "worker", ]
+      - results:/results:rw,Z
+    entrypoint: ["poetry", "run", "celery", "--app", "quarry.web.worker", "worker", ]
     extra_hosts:
     - "host.docker.internal:host-gateway"
     depends_on:
@@ -36,7 +34,7 @@ services:
   mywiki:
     image: mariadb:10.4-focal
     volumes:
-      - ./docker-replica/replica.sql:/docker-entrypoint-initdb.d/replica.sql
+      - ./docker-replica/replica.sql:/docker-entrypoint-initdb.d/replica.sql:ro,Z
     environment:
       MYSQL_USER: repl
       MYSQL_PASSWORD: repl
@@ -46,7 +44,7 @@ services:
   db:
     image: mariadb:10.1.48-bionic
     volumes:
-      - ./schema.sql:/docker-entrypoint-initdb.d/schema.sql
+      - ./schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro,Z
     environment:
       MYSQL_USER: quarry
       MYSQL_PASSWORD: quarry

helm-quarry/templates/web_deployment.yaml

@@ -18,8 +18,8 @@ spec:
         - name: web
           image: {{ .Values.web.repository }}:{{ .Values.web.tag }}
           imagePullPolicy: Always
-          command: ["gunicorn"]
-          args: ["-w", "2", "--bind", "0.0.0.0:5000", "wsgi:application"]
+          command: ["poetry"]
+          args: ["run", "gunicorn", "-w", "2", "--bind", "0.0.0.0:5000", "wsgi:application"]
           readinessProbe:
             httpGet:
               path: /

helm-quarry/templates/worker_deployment.yaml

@@ -18,8 +18,8 @@ spec:
         - name: worker
           image: {{ .Values.worker.repository }}:{{ .Values.worker.tag }}
           imagePullPolicy: Always
-          command: ["celery"]
-          args: ["--app", "quarry.web.worker", "worker"]
+          command: ["poetry"]
+          args: ["run", "celery", "--app", "quarry.web.worker", "worker"]
           volumeMounts:
             - mountPath: "/results"
               name: results

helm-quarry/values.yaml

@@ -1,6 +1,6 @@
 web:
   repository: 'quay.io/wikimedia-quarry/quarry'
-  tag: pr-96 # web tag managed by github actions
+  tag: pr-101 # web tag managed by github actions
   resources:
     requests:
       memory: "300Mi"
@@ -11,7 +11,7 @@ web:
 
 worker:
   repository: 'quay.io/wikimedia-quarry/quarry'
-  tag: pr-96 # worker tag managed by github actions
+  tag: pr-101 # worker tag managed by github actions
   resources:
     requests:
       memory: "400Mi"

maintenance/multiinstance_migration.py

@@ -11,7 +11,6 @@
 from pymysql.cursors import Cursor
 import yaml
 
-
 BATCH_SIZE = 300
 
 
@@ -53,7 +52,9 @@ def extract_dbnames(qtext: str, qdb: str, regex, db_regex) -> Optional[str]:
     return dbs[0]
 
 
-def write_execute(cursor: Cursor, query: str, dry_run: bool = False, *args) -> None:
+def write_execute(
+    cursor: Cursor, query: str, dry_run: bool = False, *args
+) -> None:
     """Do operation or simulate
     :param cursor: Cursor
     :param query: str
@@ -128,7 +129,9 @@ def main() -> None:
         conn.commit()
         cursor.execute("SELECT id, text, query_database from query_revision;")
         queries = cursor.fetchmany(BATCH_SIZE)
-        db_regex = re.compile(r"^(?:(?:centralauth|meta|[a-z]*wik[a-z]+)(?:_p)?)?$")
+        db_regex = re.compile(
+            r"^(?:(?:centralauth|meta|[a-z]*wik[a-z]+)(?:_p)?)?$"
+        )
         regex = re.compile(r"(use|USE)\s+(?P<db>\w+)\s*;")
         while queries:
             for q_id, text, query_database in queries:  # type: ignore[misc]