Merge pull request 'SCRUM-51: Bump fulfillr image to 20260614.1925 (dev + prod)' (#5) from scrum-51/fulfillr-image-bump into master

Adds email_sent/email_failed timeline events for ticket label emails.
Deployed to fulfillr-dev and fulfillr (prod) on home.debyl.io.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

ansible/deploy_home.yml

@@ -8,8 +8,6 @@
     - role: podman
 # SSL certificates are now handled automatically by Caddy
     # - role: ssl  # REMOVED - Caddy handles all certificate management
-    - role: ollama
-      tags: ollama
     - role: github-actions
     - role: graylog-config
       tags: graylog-config

ansible/roles/gitea-actions/tasks/user.yml

@@ -69,3 +69,50 @@
     group: "{{ gitea_runner_user }}"
     mode: "0644"
   tags: gitea-actions
+
+# CI jobs run in ephemeral rootless-podman containers that don't inherit the
+# gitea-runner user's ~/.ssh. Stage a dedicated, SELinux-labelled copy of the
+# runner's key + known_hosts and bind-mount it read-only into every job
+# container at /root/.ssh (see config.yaml.j2) so submodule clones over
+# ssh://git@git.skudak.com:2222 work. Kept separate from ~/.ssh so the real
+# directory's label is never touched.
+- name: create ci-ssh dir for job-container mount
+  become: true
+  ansible.builtin.file:
+    path: "{{ gitea_runner_home }}/ci-ssh"
+    state: directory
+    owner: "{{ gitea_runner_user }}"
+    group: "{{ gitea_runner_user }}"
+    mode: "0700"
+  tags: gitea-actions
+
+- name: stage runner ssh material into ci-ssh
+  become: true
+  ansible.builtin.copy:
+    src: "{{ gitea_runner_home }}/.ssh/{{ item.name }}"
+    dest: "{{ gitea_runner_home }}/ci-ssh/{{ item.name }}"
+    remote_src: true
+    owner: "{{ gitea_runner_user }}"
+    group: "{{ gitea_runner_user }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { name: id_ed25519, mode: "0600" }
+    - { name: known_hosts, mode: "0644" }
+  notify: restart act_runner services
+  tags: gitea-actions
+
+- name: label ci-ssh as container_file_t so job containers can read it
+  become: true
+  community.general.sefcontext:
+    target: "{{ gitea_runner_home }}/ci-ssh(/.*)?"
+    setype: container_file_t
+    state: present
+  register: ci_ssh_sefcontext
+  tags: gitea-actions
+
+- name: apply selinux label to ci-ssh
+  become: true
+  ansible.builtin.command: restorecon -RF {{ gitea_runner_home }}/ci-ssh
+  when: ci_ssh_sefcontext is changed
+  changed_when: true
+  tags: gitea-actions

ansible/roles/gitea-actions/templates/Containerfile.ci

@@ -7,7 +7,8 @@ ARG DOCKER_CLI_VERSION=27.3.1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         ca-certificates curl git openssh-client make build-essential \
-        python3 python3-pip jq unzip \
+        python3 python3-pip python3-yaml python3-jinja2 jq zip unzip \
+        gcc-arm-none-eabi binutils-arm-none-eabi libnewlib-arm-none-eabi \
     && rm -rf /var/lib/apt/lists/*
 
 # Static docker client (no daemon) for jobs that run `docker build` against the
@@ -22,3 +23,13 @@ RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tm
     && unzip -q /tmp/awscliv2.zip -d /tmp \
     && /tmp/aws/install \
     && rm -rf /tmp/aws /tmp/awscliv2.zip
+
+# Terraform via tfenv — workflows can pin a version with a .terraform-version
+# file (or TFENV_TERRAFORM_VERSION); the image ships "latest" as the default.
+ENV TFENV_ROOT=/opt/tfenv
+ARG TFENV_TERRAFORM_VERSION=latest
+RUN git clone --depth=1 https://github.com/tfutils/tfenv.git "${TFENV_ROOT}" \
+    && ln -s "${TFENV_ROOT}/bin/tfenv" /usr/local/bin/tfenv \
+    && ln -s "${TFENV_ROOT}/bin/terraform" /usr/local/bin/terraform \
+    && tfenv install "${TFENV_TERRAFORM_VERSION}" \
+    && tfenv use "${TFENV_TERRAFORM_VERSION}"

ansible/roles/gitea-actions/templates/config.yaml.j2

@@ -23,9 +23,16 @@ container:
   # per-job Go module/build caches and fixes cross-repo cache poisoning.
   network: host
   privileged: false
-  options:
+  # Bind-mount the runner's SSH material (key + known_hosts) read-only into
+  # every job container at /root/.ssh (CI image runs as root) so git submodule
+  # clones over ssh://git@git.skudak.com:2222 succeed. ci-ssh is a dedicated
+  # container_file_t-labelled copy staged in tasks/user.yml.
+  options: -v {{ gitea_runner_home }}/ci-ssh:/root/.ssh:ro
   workdir_parent:
-  valid_volumes: []
+  # act_runner gates host bind-mounts against this allowlist; the ci-ssh source
+  # path must be listed or the -v above is silently stripped from the job container.
+  valid_volumes:
+    - {{ gitea_runner_home }}/ci-ssh
   # Point act at the real rootless socket so it mounts the correct path into
   # job containers (the documented rootless-podman gotcha).
   docker_host: "unix:///run/user/{{ gitea_runner_uid }}/podman/podman.sock"

ansible/roles/ollama/defaults/main.yml

@@ -1,6 +0,0 @@
----
-ollama_models:
-  - dolphin-phi
-  - dolphin-mistral
-ollama_host: "127.0.0.1"
-ollama_port: 11434

ansible/roles/ollama/handlers/main.yml

@@ -1,8 +0,0 @@
----
-- name: restart ollama
-  become: true
-  ansible.builtin.systemd:
-    name: ollama
-    state: restarted
-    daemon_reload: true
-  tags: ollama

ansible/roles/ollama/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - role: common

ansible/roles/ollama/tasks/install.yml

@@ -1,11 +0,0 @@
----
-- name: check if ollama is already installed
-  ansible.builtin.stat:
-    path: /usr/local/bin/ollama
-  register: ollama_binary
-
-- name: install ollama via official install script
-  become: true
-  ansible.builtin.shell: |
-    curl -fsSL https://ollama.com/install.sh | sh
-  when: not ollama_binary.stat.exists

ansible/roles/ollama/tasks/main.yml

@@ -1,9 +0,0 @@
----
-- import_tasks: install.yml
-  tags: ollama
-
-- import_tasks: service.yml
-  tags: ollama
-
-- import_tasks: models.yml
-  tags: ollama

ansible/roles/ollama/tasks/models.yml

@@ -1,10 +0,0 @@
----
-- name: pull ollama models
-  become: true
-  ansible.builtin.command: ollama pull {{ item }}
-  loop: "{{ ollama_models }}"
-  register: result
-  retries: 3
-  delay: 10
-  until: result is not failed
-  changed_when: "'pulling' in result.stderr or 'pulling' in result.stdout"

ansible/roles/ollama/tasks/service.yml

@@ -1,23 +0,0 @@
----
-- name: create ollama systemd override directory
-  become: true
-  ansible.builtin.file:
-    path: /etc/systemd/system/ollama.service.d
-    state: directory
-    mode: 0755
-
-- name: template ollama environment override
-  become: true
-  ansible.builtin.template:
-    src: ollama.env.j2
-    dest: /etc/systemd/system/ollama.service.d/override.conf
-    mode: 0644
-  notify: restart ollama
-
-- name: enable and start ollama service
-  become: true
-  ansible.builtin.systemd:
-    name: ollama
-    enabled: true
-    state: started
-    daemon_reload: true

ansible/roles/ollama/templates/ollama.env.j2

@@ -1,4 +0,0 @@
-[Service]
-Environment="OLLAMA_HOST={{ ollama_host }}:{{ ollama_port }}"
-Environment="OLLAMA_NUM_PARALLEL=1"
-Environment="OLLAMA_MAX_LOADED_MODELS=1"

ansible/roles/podman/defaults/main.yml

@@ -10,11 +10,8 @@ fulfillr_path: "{{ podman_volumes }}/fulfillr"
 fulfillr_cases_table: "debyltech-cases-prod"
 fulfillr_tickets_table: "debyltech-tickets-prod"
 # Turso ecommerce store (self-hosted checkout).
-# PROD store is OFF until cutover (empty URL -> store routes not registered). At
-# cutover set this to libsql://debyltech-store-prod-debyltech.aws-us-east-1.turso.io
-# and add `fulfillr_store_auth_token` (prod RW token) to the vault.
-fulfillr_store_database_url: ""
-fulfillr_store_auth_token: ""
+# PROD store URL (non-secret); the RW token `fulfillr_prod_store_auth_token` is in the vault.
+fulfillr_prod_store_database_url: "libsql://debyltech-store-prod-debyltech.aws-us-east-1.turso.io"
 # Staging back-office (fulfillr-dev.debyltech.com, port 9055) -> staging Turso store.
 # Its RW token is `fulfillr_dev_store_auth_token` and EasyPost test key is
 # `fulfillr_dev_easypost_api_key`, both in the encrypted vault.
@@ -22,7 +19,6 @@ fulfillr_dev_path: "{{ podman_volumes }}/fulfillr-dev"
 fulfillr_dev_server_name: fulfillr-dev.debyltech.com
 fulfillr_dev_store_database_url: "libsql://debyltech-store-staging-debyltech.aws-us-east-1.turso.io"
 gregtime_path: "{{ podman_volumes }}/gregtime"
-searxng_path: "{{ podman_volumes }}/searxng"
 hass_path: "{{ podman_volumes }}/hass"
 # nginx_path: removed - nginx no longer used
 # nosql_path: removed - nosql/redis no longer used

ansible/roles/podman/tasks/containers/home/gregtime.yml

@@ -40,14 +40,8 @@
       - host
     env:
       TZ: America/New_York
-      # Ollama + SearXNG for FISTO AI responses
-      OLLAMA_HOST: "http://127.0.0.1:11434"
-      OLLAMA_MODEL: "dolphin-mistral"
-      OLLAMA_FALLBACK_MODEL: "dolphin-phi"
-      OLLAMA_NUM_PREDICT: "300"
-      SEARXNG_URL: "http://127.0.0.1:8080"
-      # Gemini API for @bot gemini command
-      GEMINI_API_KEY: "{{ gemini_api_key }}"
+      # xAI Grok API — the bot's sole AI backend
+      XAI_API_KEY: "{{ xai_api_key }}"
       # Zomboid RCON configuration for Discord restart command
       ZOMBOID_RCON_HOST: "127.0.0.1"
       ZOMBOID_RCON_PORT: "{{ zomboid_rcon_port }}"

ansible/roles/podman/tasks/containers/home/searxng.yml

@@ -1,59 +0,0 @@
----
-- name: create searxng host directory volumes
-  become: true
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    owner: "{{ podman_subuid.stdout }}"
-    group: "{{ podman_user }}"
-    mode: 0755
-  notify: restorecon podman
-  loop:
-    - "{{ searxng_path }}/config"
-    - "{{ searxng_path }}/data"
-
-- name: template searxng settings
-  become: true
-  ansible.builtin.template:
-    src: searxng/settings.yml.j2
-    dest: "{{ searxng_path }}/config/settings.yml"
-    owner: "{{ podman_subuid.stdout }}"
-    group: "{{ podman_user }}"
-    mode: 0644
-
-- name: unshare chown the searxng volumes for internal uid 977
-  become: true
-  become_user: "{{ podman_user }}"
-  changed_when: false
-  ansible.builtin.shell: |
-    podman unshare chown -R 977:977 {{ searxng_path }}/config
-    podman unshare chown -R 977:977 {{ searxng_path }}/data
-
-- name: flush handlers
-  ansible.builtin.meta: flush_handlers
-
-- import_tasks: podman/podman-check.yml
-  vars:
-    container_name: searxng
-    container_image: "{{ image }}"
-
-- name: create searxng container
-  become: true
-  become_user: "{{ podman_user }}"
-  containers.podman.podman_container:
-    name: searxng
-    image: "{{ image }}"
-    restart_policy: on-failure:3
-    log_driver: journald
-    network:
-      - host
-    env:
-      SEARXNG_BASE_URL: "http://127.0.0.1:8080/"
-    volumes:
-      - "{{ searxng_path }}/config:/etc/searxng"
-      - "{{ searxng_path }}/data:/srv/searxng/data"
-
-- name: create systemd startup job for searxng
-  include_tasks: podman/systemd-generate.yml
-  vars:
-    container_name: searxng

ansible/roles/podman/tasks/main.yml

@@ -78,13 +78,13 @@
 
 - import_tasks: containers/debyltech/fulfillr.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260605.2021
+    image: git.debyl.io/debyltech/fulfillr:20260614.1925
   tags: debyltech, fulfillr
 
 # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
 - import_tasks: containers/debyltech/fulfillr-dev.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260606.0357
+    image: git.debyl.io/debyltech/fulfillr:20260614.1925
   tags: debyltech, fulfillr-dev
 
 - import_tasks: containers/debyltech/uptime-kuma.yml
@@ -107,14 +107,9 @@
     image: docker.io/graylog/graylog:7.0.1
   tags: debyltech, graylog
 
-- import_tasks: containers/home/searxng.yml
-  vars:
-    image: docker.io/searxng/searxng:latest
-  tags: searxng
-
 - import_tasks: containers/home/gregtime.yml
   vars:
-    image: localhost/greg-time-bot:3.6.5
+    image: localhost/greg-time-bot:3.9.6
   tags: gregtime
 
 - import_tasks: containers/home/zomboid.yml

ansible/roles/podman/templates/fulfillr/dev.json.j2

@@ -2,10 +2,9 @@
    - ecommerce store -> STAGING Turso (fulfillr_dev_store_*)
    - EasyPost + Stripe -> TEST keys (fulfillr_dev_easypost_api_key / fulfillr_dev_stripe_api_key)
    - AWS -> FulfillrAPI-Dev key (fulfillr_dev_access_key/secret_key), scoped to the -dev
-     DynamoDB tables + debyltech.reviewr.dev. Snipcart key + outreach HMAC secret are
-     reused read-only. Never touches prod data or live payment APIs. #}
+     DynamoDB tables + debyltech.reviewr.dev. Outreach HMAC secret is reused read-only.
+     Never touches prod data or live payment APIs. (Snipcart removed post-migration.) #}
 {
-  "snipcart_api_key": "{{ snipcart_api_key }}",
   "easypost_api_key": "{{ fulfillr_dev_easypost_api_key }}",
   "stripe_api_key": "{{ fulfillr_dev_stripe_api_key }}",
   "backinstock_table": "debyltech-backinstock-dev",
@@ -13,11 +12,12 @@
   "tickets_table": "debyltech-tickets-dev",
   "store_database_url": "{{ fulfillr_dev_store_database_url }}",
   "store_auth_token": "{{ fulfillr_dev_store_auth_token }}",
+  "download_base_url": "https://api-dev.debyltech.com",
   "aws": {
     "access_key": "{{ fulfillr_dev_access_key }}",
     "secret_key": "{{ fulfillr_dev_secret_key }}",
     "region": "{{ fulfillr_region }}",
-    "bucket": "debyltech.reviewr.dev"
+    "bucket": "debyltech.digital.dev"
   },
   "tax": {
     "ein": "{{ fulfillr_tax_ein }}",
@@ -43,6 +43,12 @@
     "ses_from_email": "noreply@debyltech.com",
     "ses_reply_to": "support@debyltech.com",
     "ses_region": "us-east-1",
-    "base_url": "https://debyltech.com"
+    "base_url": "https://debyltech.com",
+    "schedule_name": "review-outreach-dev",
+    "schedule_group": "default"
+  },
+  "recovery": {
+    "schedule_name": "cart-recovery-dev",
+    "schedule_group": "default"
   }
 }

ansible/roles/podman/templates/fulfillr/production.json.j2

@@ -1,17 +1,23 @@
+{# Production back-office config (fulfillr). Live tier:
+   - ecommerce store -> PROD Turso (fulfillr_prod_store_*)
+   - EasyPost + Stripe -> LIVE keys (fulfillr_prod_easypost_api_key / fulfillr_prod_stripe_api_key)
+   - AWS -> Fulfillr prod key (fulfillr_prod_access_key/secret_key), prod DynamoDB tables +
+     debyltech.digital.prod. fulfillr_region, fulfillr_tax_ein and fulfillr_hmac_arn are
+     shared vars (no dev/prod split). Mirrors dev.json.j2. (Snipcart removed post-migration.) #}
 {
-  "snipcart_api_key": "{{ snipcart_api_key }}",
   "easypost_api_key": "{{ easypost_api_key }}",
-  "stripe_api_key": "{{ fulfillr_stripe_api_key }}",
-  "backinstock_table": "{{ fulfillr_backinstock_table }}",
-  "cases_table": "{{ fulfillr_cases_table }}",
-  "tickets_table": "{{ fulfillr_tickets_table }}",
-  "store_database_url": "{{ fulfillr_store_database_url }}",
-  "store_auth_token": "{{ fulfillr_store_auth_token }}",
+  "stripe_api_key": "{{ fulfillr_prod_stripe_api_key }}",
+  "backinstock_table": "debyltech-backinstock-prod",
+  "cases_table": "debyltech-cases-prod",
+  "tickets_table": "debyltech-tickets-prod",
+  "store_database_url": "{{ fulfillr_prod_store_database_url }}",
+  "store_auth_token": "{{ fulfillr_prod_store_auth_token }}",
+  "download_base_url": "https://api.debyltech.com",
   "aws": {
-    "access_key": "{{ fulfillr_access_key }}",
-    "secret_key": "{{ fulfillr_secret_key }}",
+    "access_key": "{{ fulfillr_prod_access_key }}",
+    "secret_key": "{{ fulfillr_prod_secret_key }}",
     "region": "{{ fulfillr_region }}",
-    "bucket": "{{ fulfillr_bucket }}"
+    "bucket": "debyltech.digital.prod"
   },
   "tax": {
     "ein": "{{ fulfillr_tax_ein }}",
@@ -37,6 +43,12 @@
     "ses_from_email": "noreply@debyltech.com",
     "ses_reply_to": "support@debyltech.com",
     "ses_region": "us-east-1",
-    "base_url": "https://debyltech.com"
+    "base_url": "https://debyltech.com",
+    "schedule_name": "review-outreach-prod",
+    "schedule_group": "default"
+  },
+  "recovery": {
+    "schedule_name": "cart-recovery-prod",
+    "schedule_group": "default"
   }
-}
+}

ansible/roles/podman/templates/searxng/settings.yml.j2

@@ -1,35 +0,0 @@
-use_default_settings: true
-
-general:
-  instance_name: "SearXNG"
-  debug: false
-
-server:
-  bind_address: "127.0.0.1"
-  port: 8080
-  secret_key: "{{ searxng_secret_key }}"
-  limiter: false
-  image_proxy: false
-
-search:
-  safe_search: 0
-  formats:
-    - html
-    - json
-
-engines:
-  - name: duckduckgo
-    engine: duckduckgo
-    disabled: false
-
-  - name: google
-    engine: google
-    disabled: false
-
-  - name: wikipedia
-    engine: wikipedia
-    disabled: false
-
-  - name: bing
-    engine: bing
-    disabled: false