Merge pull request 'SCRUM-51: Bump fulfillr image to 20260614.1925 (dev + prod)' (#5) from scrum-51/fulfillr-image-bump into master

Adds email_sent/email_failed timeline events for ticket label emails.
Deployed to fulfillr-dev and fulfillr (prod) on home.debyl.io.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

TODO.md

@@ -1,52 +0,0 @@
-# TODO
-
-## ✅ Caddy Migration - COMPLETED
-- [x] Migrate from nginx + ModSecurity to Caddy
-- [x] Automatic HTTPS certificate provisioning
-- [x] All sites working with proper IP restrictions
-- [x] Remove migration_mode logic - Caddy is now default
-
-## Infrastructure Cleanup Tasks
-
-### ✅ Phase 1: System LetsEncrypt to Caddy Migration - COMPLETED
-- [x] ~~Create dedicated Caddy certificates volume~~ - Not needed, Caddy manages in /data
-- [x] ~~Copy existing system LetsEncrypt certificates~~ - Not needed, Caddy generated new ones
-- [x] ~~Set proper permissions~~ - Already correct, Caddy runs as podman user
-- [x] Remove LetsEncrypt cron jobs from Ansible (cleanup.yml created)
-- [x] Remove LetsEncrypt cron jobs from remote host (both weekly + 5min jobs removed)
-- [x] Disable ssl role tasks and certificate generation (disabled in deploy_home.yml)
-- [x] ~~Remove certbot installation from common role~~ - Not installed there
-- [x] Uninstall certbot/letsencrypt packages from remote host (removed via dnf)
-- [x] Stop any running LetsEncrypt services (certbot.timer not running)
-- [x] Backup and remove /etc/letsencrypt directory (backup created, directory removed)
-- [x] Remove /srv/http/letsencrypt directory (webroot removed)
-
-### ✅ Phase 2: nginx + ModSecurity Cleanup - COMPLETED
-- [x] Remove nginx container configuration and tasks (deleted all conf-nginx*.yml, nginx.yml)
-- [x] Remove nginx configuration templates and files (removed entire templates/nginx/ directory)
-- [x] Remove ModSecurity rules and configuration (removed from defaults/main.yml variables)
-- [x] Remove nginx/ModSecurity volume mounts and directories (nginx volume backed up and removed)
-- [x] Clean up nginx-related variables from defaults/main.yml (nginx_path removed)
-- [x] ~~Remove firewall rules for nginx~~ - Not needed, Caddy uses same ports
-- [x] Remove nginx systemd services from remote host (container-nginx service removed)
-- [x] ~~Uninstall nginx/ModSecurity packages~~ - Were never system-installed, container-only
-- [x] Clean up nginx log directories and files (/var/log/nginx, /var/log/modsecurity removed)
-- [x] Remove ModSecurity installation directories (/usr/share/modsecurity, /usr/share/coreruleset removed)
-- [x] Create backup of nginx configuration (nginx-backup-{timestamp}.tar.gz created)
-
-### ✅ Phase 3: Final Cleanup - COMPLETED
-- [x] Remove Drone CI infrastructure and ci.bdebyl.net host
-  - [x] Remove Drone container from podman configuration (drone.yml deleted)
-  - [x] Remove ci.bdebyl.net from Caddyfile (site configuration removed)
-  - [x] Clean up drone-related volumes and data (drone volume backed up and removed)
-  - [x] Update firewall rules to remove CI ports (ports were not explicitly opened)
-- [x] Review and remove unused variables and templates
-  - [x] Removed ci_server_name variable
-  - [x] Removed drone-related variables (drone_path, drone_server_proto, etc.)
-  - [x] Cleaned up nginx handler in handlers/main.yml
-  - [x] Updated firewall.yml comments
-- [x] Update documentation to reflect Caddy as web server
-  - [x] Updated CLAUDE.md container organization section
-  - [x] Updated tagging strategy (nginx→caddy, drone marked decommissioned)
-  - [x] Updated target environment description (nginx→Caddy)
-- [x] Verify all services working after cleanup (sites tested and working)

ansible/deploy_home.yml

@@ -8,8 +8,6 @@
     - role: podman
 # SSL certificates are now handled automatically by Caddy
     # - role: ssl  # REMOVED - Caddy handles all certificate management
-    - role: ollama
-      tags: ollama
     - role: github-actions
     - role: graylog-config
       tags: graylog-config

ansible/roles/gitea-actions/defaults/main.yml

@@ -3,23 +3,35 @@ gitea_runner_user: gitea-runner
 gitea_runner_home: /home/gitea-runner
 gitea_runner_version: "0.2.13"
 gitea_runner_arch: linux-amd64
+
+# Max concurrent jobs per runner. Each job runs in its own ephemeral container
+# (docker:// labels backed by rootless podman), so jobs no longer share the
+# gitea-runner user's Go caches and can run fully in parallel without corruption.
 gitea_runner_capacity: 4
 
-# Multiple Gitea instances to run actions runners for
+# Gitea instances to run actions runners for. Override `labels` or `capacity`
+# per runner here if needed.
 gitea_runners:
   - name: debyl
     instance_url: https://git.debyl.io
   - name: skudak
     instance_url: https://git.skudak.com
 
-# Old single-instance format (replaced by gitea_runners list above):
-# gitea_instance_url: https://git.debyl.io
-
 # Paths
 act_runner_bin: /usr/local/bin/act_runner
 act_runner_config_dir: /etc/act_runner
 act_runner_work_dir: /var/lib/act_runner
 
-# ESP-IDF configuration
+# Job container images (built locally into the gitea-runner rootless image
+# store by tasks/images.yml; never pulled — force_pull is false).
+gitea_ci_image: localhost/gitea-ci:latest
+# ESP-IDF firmware image tag tracks the upstream espressif/idf release we build from.
 esp_idf_version: v5.4.1
-esp_idf_path: /opt/esp-idf
+gitea_ci_espidf_image: "localhost/gitea-ci-espidf:{{ esp_idf_version }}"
+
+# Default labels for every runner — map runs-on values to the local CI image.
+# Firmware jobs opt into the ESP-IDF image per-job via `container:` in their workflow.
+gitea_runner_labels:
+  - "fedora:docker://{{ gitea_ci_image }}"
+  - "ubuntu-latest:docker://{{ gitea_ci_image }}"
+  - "ubuntu-22.04:docker://{{ gitea_ci_image }}"

ansible/roles/gitea-actions/handlers/main.yml

@@ -6,16 +6,3 @@
     state: restarted
     daemon_reload: true
   loop: "{{ gitea_runners }}"
-
-- name: restart podman socket
-  become: true
-  ansible.builtin.systemd:
-    name: podman.socket
-    state: restarted
-    daemon_reload: true
-
-- name: restore esp-idf selinux context
-  become: true
-  ansible.builtin.command:
-    cmd: restorecon -R {{ esp_idf_path }}
-  changed_when: true

ansible/roles/gitea-actions/tasks/deps.yml

@@ -1,38 +1,69 @@
 ---
-- name: install podman-docker for docker CLI compatibility
+- name: install podman for rootless CI job containers
   become: true
   ansible.builtin.dnf:
     name:
-      - podman-docker
-      - golang
+      - podman
     state: present
   tags: gitea-actions
 
-- name: create podman socket override directory
+- name: look up gitea-runner uid
   become: true
-  ansible.builtin.file:
-    path: /etc/systemd/system/podman.socket.d
-    state: directory
-    mode: "0755"
+  changed_when: false
+  check_mode: false
+  ansible.builtin.command: id -u {{ gitea_runner_user }}
+  register: gitea_runner_id
+  tags:
+    - gitea-actions
+    - always
+
+- name: set gitea_runner_uid fact
+  ansible.builtin.set_fact:
+    gitea_runner_uid: "{{ gitea_runner_id.stdout | trim }}"
+  tags:
+    - gitea-actions
+    - always
+
+# Rootless podman needs subuid/subgid ranges for the runner user. Fedora's
+# useradd normally assigns them automatically; ensure they exist regardless.
+- name: check gitea-runner subuid mapping
+  become: true
+  ansible.builtin.command: grep -q "^{{ gitea_runner_user }}:" /etc/subuid
+  register: gitea_runner_subuid
+  changed_when: false
+  failed_when: false
   tags: gitea-actions
 
-- name: configure podman socket for gitea-runner access
+- name: assign subuid/subgid ranges for gitea-runner
   become: true
-  ansible.builtin.copy:
-    dest: /etc/systemd/system/podman.socket.d/override.conf
-    content: |
-      [Socket]
-      SocketMode=0660
-      SocketGroup={{ gitea_runner_user }}
-    mode: "0644"
-  notify: restart podman socket
+  ansible.builtin.command: >-
+    usermod
+    --add-subuids 100000000-100065535
+    --add-subgids 100000000-100065535
+    {{ gitea_runner_user }}
+  when: gitea_runner_subuid.rc != 0
+  register: gitea_runner_subuid_added
   tags: gitea-actions
 
-- name: enable system podman socket
+- name: migrate gitea-runner podman storage to new id mapping
   become: true
+  become_user: "{{ gitea_runner_user }}"
+  ansible.builtin.command: podman system migrate
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
+  when: gitea_runner_subuid_added is changed
+  changed_when: true
+  tags: gitea-actions
+
+- name: enable rootless podman socket for gitea-runner
+  become: true
+  become_user: "{{ gitea_runner_user }}"
   ansible.builtin.systemd:
     name: podman.socket
-    daemon_reload: true
+    scope: user
     enabled: true
     state: started
+    daemon_reload: true
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
   tags: gitea-actions

ansible/roles/gitea-actions/tasks/esp-idf.yml

@@ -1,92 +0,0 @@
----
-- name: install ESP-IDF build dependencies
-  become: true
-  ansible.builtin.dnf:
-    name:
-      - git
-      - wget
-      - flex
-      - bison
-      - gperf
-      - python3
-      - python3-pip
-      - cmake
-      - ninja-build
-      - ccache
-      - libffi-devel
-      - libusb1-devel
-    state: present
-  tags: gitea-actions
-
-- name: check if ESP-IDF is installed
-  ansible.builtin.stat:
-    path: "{{ esp_idf_path }}"
-  register: esp_idf_dir
-  tags: gitea-actions
-
-- name: clone ESP-IDF repository
-  become: true
-  ansible.builtin.git:
-    repo: https://github.com/espressif/esp-idf.git
-    dest: "{{ esp_idf_path }}"
-    version: "{{ esp_idf_version }}"
-    recursive: true
-    force: false
-  when: not esp_idf_dir.stat.exists
-  tags: gitea-actions
-
-- name: add ESP-IDF to git safe.directory
-  become: true
-  ansible.builtin.command:
-    cmd: git config --global --add safe.directory {{ esp_idf_path }}
-  changed_when: false
-  tags: gitea-actions
-
-- name: ensure ESP-IDF submodules are initialized
-  become: true
-  ansible.builtin.command:
-    cmd: git submodule update --init --recursive
-    chdir: "{{ esp_idf_path }}"
-  changed_when: false
-  tags: gitea-actions
-
-- name: set ESP-IDF directory ownership
-  become: true
-  ansible.builtin.file:
-    path: "{{ esp_idf_path }}"
-    owner: "{{ gitea_runner_user }}"
-    group: "{{ gitea_runner_user }}"
-    recurse: true
-  tags: gitea-actions
-
-- name: set SELinux context for ESP-IDF directory
-  become: true
-  community.general.sefcontext:
-    target: "{{ esp_idf_path }}(/.*)?"
-    setype: usr_t
-    state: present
-  when: ansible_selinux.status == "enabled"
-  notify: restore esp-idf selinux context
-  tags: gitea-actions
-
-- name: create ESP-IDF tools directory for runner user
-  become: true
-  ansible.builtin.file:
-    path: "{{ gitea_runner_home }}/.espressif"
-    state: directory
-    owner: "{{ gitea_runner_user }}"
-    group: "{{ gitea_runner_user }}"
-    mode: "0755"
-  tags: gitea-actions
-
-- name: install ESP-IDF tools for runner user
-  become: true
-  become_user: "{{ gitea_runner_user }}"
-  ansible.builtin.shell: |
-    export IDF_TOOLS_PATH="{{ gitea_runner_home }}/.espressif"
-    {{ esp_idf_path }}/install.sh esp32
-  args:
-    creates: "{{ gitea_runner_home }}/.espressif/python_env"
-  environment:
-    HOME: "{{ gitea_runner_home }}"
-  tags: gitea-actions

ansible/roles/gitea-actions/tasks/images.yml

@@ -0,0 +1,55 @@
+---
+- name: create CI image build directory
+  become: true
+  become_user: "{{ gitea_runner_user }}"
+  ansible.builtin.file:
+    path: "{{ gitea_runner_home }}/ci-images"
+    state: directory
+    mode: "0755"
+  tags: gitea-actions
+
+- name: stage default CI Containerfile
+  become: true
+  become_user: "{{ gitea_runner_user }}"
+  ansible.builtin.template:
+    src: Containerfile.ci
+    dest: "{{ gitea_runner_home }}/ci-images/Containerfile.ci"
+    mode: "0644"
+  register: ci_containerfile
+  tags: gitea-actions
+
+- name: stage ESP-IDF CI Containerfile
+  become: true
+  become_user: "{{ gitea_runner_user }}"
+  ansible.builtin.template:
+    src: Containerfile.espidf.j2
+    dest: "{{ gitea_runner_home }}/ci-images/Containerfile.espidf"
+    mode: "0644"
+  register: espidf_containerfile
+  tags: gitea-actions
+
+- name: build default CI image ({{ gitea_ci_image }})
+  become: true
+  become_user: "{{ gitea_runner_user }}"
+  containers.podman.podman_image:
+    name: "{{ gitea_ci_image }}"
+    path: "{{ gitea_runner_home }}/ci-images"
+    build:
+      file: "{{ gitea_runner_home }}/ci-images/Containerfile.ci"
+    force: "{{ ci_containerfile is changed }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
+  tags: gitea-actions
+
+- name: build ESP-IDF CI image ({{ gitea_ci_espidf_image }})
+  become: true
+  become_user: "{{ gitea_runner_user }}"
+  containers.podman.podman_image:
+    name: "{{ gitea_ci_espidf_image }}"
+    path: "{{ gitea_runner_home }}/ci-images"
+    build:
+      file: "{{ gitea_runner_home }}/ci-images/Containerfile.espidf"
+    force: "{{ espidf_containerfile is changed }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
+  tags: gitea-actions

ansible/roles/gitea-actions/tasks/main.yml

@@ -3,7 +3,7 @@
   tags: gitea-actions
 - import_tasks: deps.yml
   tags: gitea-actions
-- import_tasks: esp-idf.yml
+- import_tasks: images.yml
   tags: gitea-actions
 - import_tasks: runner.yml
   tags: gitea-actions

ansible/roles/gitea-actions/tasks/runner.yml

@@ -45,6 +45,8 @@
     mode: "0644"
   vars:
     runner_name: "{{ item.name }}"
+    runner_capacity: "{{ item.capacity | default(gitea_runner_capacity) }}"
+    runner_labels: "{{ item.labels | default(gitea_runner_labels) }}"
   loop: "{{ gitea_runners }}"
   notify: restart act_runner services
   tags: gitea-actions

ansible/roles/gitea-actions/tasks/user.yml

@@ -7,8 +7,6 @@
     shell: /bin/bash
     createhome: true
     home: "{{ gitea_runner_home }}"
-    groups: docker
-    append: true
   tags: gitea-actions
 
 - name: check if gitea-runner lingering enabled
@@ -71,3 +69,50 @@
     group: "{{ gitea_runner_user }}"
     mode: "0644"
   tags: gitea-actions
+
+# CI jobs run in ephemeral rootless-podman containers that don't inherit the
+# gitea-runner user's ~/.ssh. Stage a dedicated, SELinux-labelled copy of the
+# runner's key + known_hosts and bind-mount it read-only into every job
+# container at /root/.ssh (see config.yaml.j2) so submodule clones over
+# ssh://git@git.skudak.com:2222 work. Kept separate from ~/.ssh so the real
+# directory's label is never touched.
+- name: create ci-ssh dir for job-container mount
+  become: true
+  ansible.builtin.file:
+    path: "{{ gitea_runner_home }}/ci-ssh"
+    state: directory
+    owner: "{{ gitea_runner_user }}"
+    group: "{{ gitea_runner_user }}"
+    mode: "0700"
+  tags: gitea-actions
+
+- name: stage runner ssh material into ci-ssh
+  become: true
+  ansible.builtin.copy:
+    src: "{{ gitea_runner_home }}/.ssh/{{ item.name }}"
+    dest: "{{ gitea_runner_home }}/ci-ssh/{{ item.name }}"
+    remote_src: true
+    owner: "{{ gitea_runner_user }}"
+    group: "{{ gitea_runner_user }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { name: id_ed25519, mode: "0600" }
+    - { name: known_hosts, mode: "0644" }
+  notify: restart act_runner services
+  tags: gitea-actions
+
+- name: label ci-ssh as container_file_t so job containers can read it
+  become: true
+  community.general.sefcontext:
+    target: "{{ gitea_runner_home }}/ci-ssh(/.*)?"
+    setype: container_file_t
+    state: present
+  register: ci_ssh_sefcontext
+  tags: gitea-actions
+
+- name: apply selinux label to ci-ssh
+  become: true
+  ansible.builtin.command: restorecon -RF {{ gitea_runner_home }}/ci-ssh
+  when: ci_ssh_sefcontext is changed
+  changed_when: true
+  tags: gitea-actions

ansible/roles/gitea-actions/templates/Containerfile.ci

@@ -0,0 +1,35 @@
+# Default Gitea Actions job image (managed by ansible: roles/gitea-actions).
+# Covers Go/web/node jobs plus `docker build` (talks to the mounted rootless
+# podman socket). Go toolchains are provided per-job by actions/setup-go.
+FROM node:20-bookworm-slim
+
+ARG DOCKER_CLI_VERSION=27.3.1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl git openssh-client make build-essential \
+        python3 python3-pip python3-yaml python3-jinja2 jq zip unzip \
+        gcc-arm-none-eabi binutils-arm-none-eabi libnewlib-arm-none-eabi \
+    && rm -rf /var/lib/apt/lists/*
+
+# Static docker client (no daemon) for jobs that run `docker build` against the
+# mounted podman socket (/var/run/docker.sock).
+RUN curl -fsSL "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" \
+        | tar -xz -C /tmp \
+    && install -m0755 /tmp/docker/docker /usr/local/bin/docker \
+    && rm -rf /tmp/docker
+
+# AWS CLI v2 — several workflows upload artifacts / deploy Lambda.
+RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \
+    && unzip -q /tmp/awscliv2.zip -d /tmp \
+    && /tmp/aws/install \
+    && rm -rf /tmp/aws /tmp/awscliv2.zip
+
+# Terraform via tfenv — workflows can pin a version with a .terraform-version
+# file (or TFENV_TERRAFORM_VERSION); the image ships "latest" as the default.
+ENV TFENV_ROOT=/opt/tfenv
+ARG TFENV_TERRAFORM_VERSION=latest
+RUN git clone --depth=1 https://github.com/tfutils/tfenv.git "${TFENV_ROOT}" \
+    && ln -s "${TFENV_ROOT}/bin/tfenv" /usr/local/bin/tfenv \
+    && ln -s "${TFENV_ROOT}/bin/terraform" /usr/local/bin/terraform \
+    && tfenv install "${TFENV_TERRAFORM_VERSION}" \
+    && tfenv use "${TFENV_TERRAFORM_VERSION}"

ansible/roles/gitea-actions/templates/Containerfile.espidf.j2

@@ -0,0 +1,16 @@
+# ESP-IDF firmware job image (managed by ansible: roles/gitea-actions).
+# Adds node (required by actions/checkout and other JS actions) and the AWS CLI
+# (firmware artifacts ship to S3) on top of the official Espressif toolchain.
+# IDF lives at /opt/esp/idf — firmware jobs source /opt/esp/idf/export.sh.
+FROM espressif/idf:{{ esp_idf_version }}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl ca-certificates unzip \
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \
+    && unzip -q /tmp/awscliv2.zip -d /tmp \
+    && /tmp/aws/install \
+    && rm -rf /tmp/aws /tmp/awscliv2.zip

ansible/roles/gitea-actions/templates/act_runner.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=Gitea Actions runner ({{ runner_name }})
 Documentation=https://gitea.com/gitea/act_runner
-After=network.target podman.socket
+After=network.target
 
 [Service]
 ExecStart={{ act_runner_bin }} daemon --config {{ act_runner_config_dir }}/config-{{ runner_name }}.yaml
@@ -10,7 +10,8 @@ TimeoutSec=0
 RestartSec=10
 Restart=always
 User={{ gitea_runner_user }}
-Environment="DOCKER_HOST=unix:///run/podman/podman.sock"
+Environment="XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}"
+Environment="DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/podman/podman.sock"
 
 [Install]
 WantedBy=multi-user.target

ansible/roles/gitea-actions/templates/config.yaml.j2

@@ -3,27 +3,39 @@ log:
 
 runner:
   file: {{ act_runner_work_dir }}/{{ runner_name }}/.runner
-  capacity: {{ gitea_runner_capacity | default(4) }}
+  capacity: {{ runner_capacity | default(gitea_runner_capacity) | default(4) }}
   timeout: 3h
   insecure: false
   fetch_timeout: 5s
   fetch_interval: 2s
   labels:
-    - ubuntu-latest:host
-    - ubuntu-22.04:host
-    - fedora:host
+{% for label in runner_labels | default(gitea_runner_labels) %}
+    - {{ label }}
+{% endfor %}
 
 cache:
   enabled: true
   dir: {{ act_runner_work_dir }}/{{ runner_name }}/cache
 
 container:
+  # Each job runs in its own ephemeral container (docker:// labels) backed by
+  # the gitea-runner user's rootless podman socket — this is what isolates the
+  # per-job Go module/build caches and fixes cross-repo cache poisoning.
   network: host
   privileged: false
-  options:
+  # Bind-mount the runner's SSH material (key + known_hosts) read-only into
+  # every job container at /root/.ssh (CI image runs as root) so git submodule
+  # clones over ssh://git@git.skudak.com:2222 succeed. ci-ssh is a dedicated
+  # container_file_t-labelled copy staged in tasks/user.yml.
+  options: -v {{ gitea_runner_home }}/ci-ssh:/root/.ssh:ro
   workdir_parent:
-  valid_volumes: []
-  docker_host: ""
+  # act_runner gates host bind-mounts against this allowlist; the ci-ssh source
+  # path must be listed or the -v above is silently stripped from the job container.
+  valid_volumes:
+    - {{ gitea_runner_home }}/ci-ssh
+  # Point act at the real rootless socket so it mounts the correct path into
+  # job containers (the documented rootless-podman gotcha).
+  docker_host: "unix:///run/user/{{ gitea_runner_uid }}/podman/podman.sock"
   force_pull: false
 
 host:

ansible/roles/ollama/defaults/main.yml

@@ -1,6 +0,0 @@
----
-ollama_models:
-  - dolphin-phi
-  - dolphin-mistral
-ollama_host: "127.0.0.1"
-ollama_port: 11434

ansible/roles/ollama/handlers/main.yml

@@ -1,8 +0,0 @@
----
-- name: restart ollama
-  become: true
-  ansible.builtin.systemd:
-    name: ollama
-    state: restarted
-    daemon_reload: true
-  tags: ollama

ansible/roles/ollama/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - role: common

ansible/roles/ollama/tasks/install.yml

@@ -1,11 +0,0 @@
----
-- name: check if ollama is already installed
-  ansible.builtin.stat:
-    path: /usr/local/bin/ollama
-  register: ollama_binary
-
-- name: install ollama via official install script
-  become: true
-  ansible.builtin.shell: |
-    curl -fsSL https://ollama.com/install.sh | sh
-  when: not ollama_binary.stat.exists

ansible/roles/ollama/tasks/main.yml

@@ -1,9 +0,0 @@
----
-- import_tasks: install.yml
-  tags: ollama
-
-- import_tasks: service.yml
-  tags: ollama
-
-- import_tasks: models.yml
-  tags: ollama

ansible/roles/ollama/tasks/models.yml

@@ -1,10 +0,0 @@
----
-- name: pull ollama models
-  become: true
-  ansible.builtin.command: ollama pull {{ item }}
-  loop: "{{ ollama_models }}"
-  register: result
-  retries: 3
-  delay: 10
-  until: result is not failed
-  changed_when: "'pulling' in result.stderr or 'pulling' in result.stdout"

ansible/roles/ollama/tasks/service.yml

@@ -1,23 +0,0 @@
----
-- name: create ollama systemd override directory
-  become: true
-  ansible.builtin.file:
-    path: /etc/systemd/system/ollama.service.d
-    state: directory
-    mode: 0755
-
-- name: template ollama environment override
-  become: true
-  ansible.builtin.template:
-    src: ollama.env.j2
-    dest: /etc/systemd/system/ollama.service.d/override.conf
-    mode: 0644
-  notify: restart ollama
-
-- name: enable and start ollama service
-  become: true
-  ansible.builtin.systemd:
-    name: ollama
-    enabled: true
-    state: started
-    daemon_reload: true

ansible/roles/ollama/templates/ollama.env.j2

@@ -1,4 +0,0 @@
-[Service]
-Environment="OLLAMA_HOST={{ ollama_host }}:{{ ollama_port }}"
-Environment="OLLAMA_NUM_PARALLEL=1"
-Environment="OLLAMA_MAX_LOADED_MODELS=1"

ansible/roles/podman/defaults/main.yml

@@ -9,8 +9,16 @@ factorio_path: "{{ podman_volumes }}/factorio"
 fulfillr_path: "{{ podman_volumes }}/fulfillr"
 fulfillr_cases_table: "debyltech-cases-prod"
 fulfillr_tickets_table: "debyltech-tickets-prod"
+# Turso ecommerce store (self-hosted checkout).
+# PROD store URL (non-secret); the RW token `fulfillr_prod_store_auth_token` is in the vault.
+fulfillr_prod_store_database_url: "libsql://debyltech-store-prod-debyltech.aws-us-east-1.turso.io"
+# Staging back-office (fulfillr-dev.debyltech.com, port 9055) -> staging Turso store.
+# Its RW token is `fulfillr_dev_store_auth_token` and EasyPost test key is
+# `fulfillr_dev_easypost_api_key`, both in the encrypted vault.
+fulfillr_dev_path: "{{ podman_volumes }}/fulfillr-dev"
+fulfillr_dev_server_name: fulfillr-dev.debyltech.com
+fulfillr_dev_store_database_url: "libsql://debyltech-store-staging-debyltech.aws-us-east-1.turso.io"
 gregtime_path: "{{ podman_volumes }}/gregtime"
-searxng_path: "{{ podman_volumes }}/searxng"
 hass_path: "{{ podman_volumes }}/hass"
 # nginx_path: removed - nginx no longer used
 # nosql_path: removed - nosql/redis no longer used

ansible/roles/podman/tasks/containers/base/awsddns.yml

@@ -80,6 +80,35 @@
   vars:
     container_name: awsddns-fulfillr
 
+- import_tasks: podman/podman-check.yml
+  vars:
+    container_name: awsddns-fulfillr-dev
+    container_image: "{{ image }}"
+
+# Staging back-office DNS — same Route53 zone + creds as prod fulfillr, just a
+# different hostname (-> same host IP; Caddy routes both by Host header).
+- name: create fulfillr-dev.debyltech.com awsddns server container
+  become: true
+  become_user: "{{ podman_user }}"
+  diff: false
+  containers.podman.podman_container:
+    name: awsddns-fulfillr-dev
+    image: "{{ image }}"
+    restart_policy: on-failure:3
+    log_driver: journald
+    env:
+      AWS_ZONE_TTL: 60
+      AWS_ZONE_ID: "{{ fulfillr_zone_id }}"
+      AWS_ZONE_HOSTNAME: "{{ fulfillr_dev_server_name }}"
+      AWS_ACCESS_KEY_ID: "{{ fulfillr_dns_access_key }}"
+      AWS_SECRET_ACCESS_KEY: "{{ fulfillr_dns_secret_key }}"
+      AWS_DEFAULT_REGION: "{{ fulfillr_region }}"
+
+- name: create systemd startup job for awsddns-fulfillr-dev
+  include_tasks: podman/systemd-generate.yml
+  vars:
+    container_name: awsddns-fulfillr-dev
+
 - import_tasks: podman/podman-check.yml
   vars:
     container_name: awsddns-debyl

ansible/roles/podman/tasks/containers/base/caddy.yml

@@ -25,6 +25,7 @@
 # Legacy volume mounts removed - Caddy manages certificates automatically
       # Mount static site directories
       - "/usr/local/share/fulfillr-site:/usr/local/share/fulfillr-site:ro"
+      - "/usr/local/share/fulfillr-site-dev:/usr/local/share/fulfillr-site-dev:ro"
       - "/usr/local/share/test-site:/srv/test-site:ro"
     env:
       CADDY_ADMIN: "0.0.0.0:2019"

ansible/roles/podman/tasks/containers/debyltech/fulfillr-dev.yml

@@ -0,0 +1,67 @@
+---
+# Staging back-office: a second go-fulfillr container (same image as prod) wired to
+# the STAGING Turso store + EasyPost test key via dev.json. Served at
+# fulfillr-dev.debyltech.com (Caddy -> :9055), LAN-restricted like prod.
+- import_tasks: gitea/podman-gitea-login.yml
+
+- name: create nginx fulfillr-site-dev directory
+  become: true
+  ansible.builtin.file:
+    path: /usr/local/share/fulfillr-site-dev
+    state: directory
+    owner: "fedora"
+    group: "wheel"
+    mode: 0755
+
+- name: create fulfillr-dev host directory volumes
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ podman_user }}"
+    group: "{{ podman_user }}"
+    mode: 0755
+  notify: restorecon podman
+  loop:
+    - "{{ fulfillr_dev_path }}"
+
+- name: template fulfillr-dev config
+  become: true
+  ansible.builtin.template:
+    src: "templates/fulfillr/{{ item }}.j2"
+    dest: "{{ fulfillr_dev_path }}/{{ item }}"
+    owner: "{{ podman_user }}"
+    group: "{{ podman_user }}"
+    mode: 0644
+  loop:
+    - dev.json
+  notify:
+    - restorecon podman
+
+- name: flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- import_tasks: podman/podman-check.yml
+  vars:
+    container_name: fulfillr-dev
+    container_image: "{{ image }}"
+
+- name: create fulfillr-dev server container
+  become: true
+  become_user: "{{ podman_user }}"
+  containers.podman.podman_container:
+    name: fulfillr-dev
+    image: "{{ image }}"
+    image_strict: true
+    command: --config /config/dev.json
+    restart_policy: on-failure:3
+    log_driver: journald
+    volumes:
+      - "{{ fulfillr_dev_path }}:/config"
+    ports:
+      - 9055:8080/tcp
+
+- name: create systemd startup job for fulfillr-dev
+  include_tasks: podman/systemd-generate.yml
+  vars:
+    container_name: fulfillr-dev

ansible/roles/podman/tasks/containers/home/gregtime.yml

@@ -40,14 +40,8 @@
       - host
     env:
       TZ: America/New_York
-      # Ollama + SearXNG for FISTO AI responses
-      OLLAMA_HOST: "http://127.0.0.1:11434"
-      OLLAMA_MODEL: "dolphin-mistral"
-      OLLAMA_FALLBACK_MODEL: "dolphin-phi"
-      OLLAMA_NUM_PREDICT: "300"
-      SEARXNG_URL: "http://127.0.0.1:8080"
-      # Gemini API for @bot gemini command
-      GEMINI_API_KEY: "{{ gemini_api_key }}"
+      # xAI Grok API — the bot's sole AI backend
+      XAI_API_KEY: "{{ xai_api_key }}"
       # Zomboid RCON configuration for Discord restart command
       ZOMBOID_RCON_HOST: "127.0.0.1"
       ZOMBOID_RCON_PORT: "{{ zomboid_rcon_port }}"

ansible/roles/podman/tasks/containers/home/searxng.yml

@@ -1,59 +0,0 @@
----
-- name: create searxng host directory volumes
-  become: true
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    owner: "{{ podman_subuid.stdout }}"
-    group: "{{ podman_user }}"
-    mode: 0755
-  notify: restorecon podman
-  loop:
-    - "{{ searxng_path }}/config"
-    - "{{ searxng_path }}/data"
-
-- name: template searxng settings
-  become: true
-  ansible.builtin.template:
-    src: searxng/settings.yml.j2
-    dest: "{{ searxng_path }}/config/settings.yml"
-    owner: "{{ podman_subuid.stdout }}"
-    group: "{{ podman_user }}"
-    mode: 0644
-
-- name: unshare chown the searxng volumes for internal uid 977
-  become: true
-  become_user: "{{ podman_user }}"
-  changed_when: false
-  ansible.builtin.shell: |
-    podman unshare chown -R 977:977 {{ searxng_path }}/config
-    podman unshare chown -R 977:977 {{ searxng_path }}/data
-
-- name: flush handlers
-  ansible.builtin.meta: flush_handlers
-
-- import_tasks: podman/podman-check.yml
-  vars:
-    container_name: searxng
-    container_image: "{{ image }}"
-
-- name: create searxng container
-  become: true
-  become_user: "{{ podman_user }}"
-  containers.podman.podman_container:
-    name: searxng
-    image: "{{ image }}"
-    restart_policy: on-failure:3
-    log_driver: journald
-    network:
-      - host
-    env:
-      SEARXNG_BASE_URL: "http://127.0.0.1:8080/"
-    volumes:
-      - "{{ searxng_path }}/config:/etc/searxng"
-      - "{{ searxng_path }}/data:/srv/searxng/data"
-
-- name: create systemd startup job for searxng
-  include_tasks: podman/systemd-generate.yml
-  vars:
-    container_name: searxng

ansible/roles/podman/tasks/main.yml

@@ -78,9 +78,15 @@
 
 - import_tasks: containers/debyltech/fulfillr.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260603.0222
+    image: git.debyl.io/debyltech/fulfillr:20260614.1925
   tags: debyltech, fulfillr
 
+# Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
+- import_tasks: containers/debyltech/fulfillr-dev.yml
+  vars:
+    image: git.debyl.io/debyltech/fulfillr:20260614.1925
+  tags: debyltech, fulfillr-dev
+
 - import_tasks: containers/debyltech/uptime-kuma.yml
   vars:
     image: docker.io/louislam/uptime-kuma:2.3.2
@@ -101,14 +107,9 @@
     image: docker.io/graylog/graylog:7.0.1
   tags: debyltech, graylog
 
-- import_tasks: containers/home/searxng.yml
-  vars:
-    image: docker.io/searxng/searxng:latest
-  tags: searxng
-
 - import_tasks: containers/home/gregtime.yml
   vars:
-    image: localhost/greg-time-bot:3.6.5
+    image: localhost/greg-time-bot:3.9.6
   tags: gregtime
 
 - import_tasks: containers/home/zomboid.yml

ansible/roles/podman/templates/caddy/Caddyfile.j2

@@ -389,6 +389,53 @@
 	}
 }
 
+# Fulfillr DEV/staging - {{ fulfillr_dev_server_name }} (Static + API with IP restrictions)
+{{ fulfillr_dev_server_name }} {
+{{ ip_restricted_site() }}
+
+	@api {
+		path /api/*
+	}
+
+	# Handle API requests with CORS for local development
+	handle @api {
+		header {
+			Access-Control-Allow-Origin "*"
+			Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+			Access-Control-Allow-Headers "Content-Type, Authorization, X-Requested-With"
+			Access-Control-Allow-Credentials "true"
+		}
+
+		# Handle preflight requests
+		@options {
+			method OPTIONS
+		}
+		handle @options {
+			respond "" 204
+		}
+
+		reverse_proxy localhost:9055
+	}
+
+	# Serve static files with SPA fallback
+	handle {
+		root * /usr/local/share/fulfillr-site-dev
+		try_files {path} {path}/ /index.html
+		file_server
+	}
+
+	header {
+		Strict-Transport-Security "max-age=31536000; includeSubDomains"
+		X-Content-Type-Options "nosniff"
+		Referrer-Policy "same-origin"
+	}
+
+	log {
+		output file /var/log/caddy/fulfillr-dev.log
+		format json
+	}
+}
+
 # ============================================================================
 # TEST/STAGING SITES
 # ============================================================================

ansible/roles/podman/templates/fulfillr/dev.json.j2

@@ -0,0 +1,54 @@
+{# Staging back-office config (fulfillr-dev). Isolated dev tier:
+   - ecommerce store -> STAGING Turso (fulfillr_dev_store_*)
+   - EasyPost + Stripe -> TEST keys (fulfillr_dev_easypost_api_key / fulfillr_dev_stripe_api_key)
+   - AWS -> FulfillrAPI-Dev key (fulfillr_dev_access_key/secret_key), scoped to the -dev
+     DynamoDB tables + debyltech.reviewr.dev. Outreach HMAC secret is reused read-only.
+     Never touches prod data or live payment APIs. (Snipcart removed post-migration.) #}
+{
+  "easypost_api_key": "{{ fulfillr_dev_easypost_api_key }}",
+  "stripe_api_key": "{{ fulfillr_dev_stripe_api_key }}",
+  "backinstock_table": "debyltech-backinstock-dev",
+  "cases_table": "debyltech-cases-dev",
+  "tickets_table": "debyltech-tickets-dev",
+  "store_database_url": "{{ fulfillr_dev_store_database_url }}",
+  "store_auth_token": "{{ fulfillr_dev_store_auth_token }}",
+  "download_base_url": "https://api-dev.debyltech.com",
+  "aws": {
+    "access_key": "{{ fulfillr_dev_access_key }}",
+    "secret_key": "{{ fulfillr_dev_secret_key }}",
+    "region": "{{ fulfillr_region }}",
+    "bucket": "debyltech.digital.dev"
+  },
+  "tax": {
+    "ein": "{{ fulfillr_tax_ein }}",
+    "ioss": null
+  },
+  "sender_address": {
+    "city": "Newbury",
+    "country": "US",
+    "email": "sales@debyltech.com",
+    "name": "de Byl Technologies LLC",
+    "phone": "6034160859",
+    "state": "NH",
+    "street1": "976 Route 103",
+    "street2": "Unit 95",
+    "zip": "03255"
+  },
+  "outreach": {
+    "outreach_table": "debyltech-outreach-dev",
+    "unsubscribe_table": "debyltech-unsubscribe-dev",
+    "email_log_table": "debyltech-email-log-dev",
+    "reviews_table": "debyltech-reviews-dev",
+    "hmac_secret_arn": "{{ fulfillr_hmac_arn }}",
+    "ses_from_email": "noreply@debyltech.com",
+    "ses_reply_to": "support@debyltech.com",
+    "ses_region": "us-east-1",
+    "base_url": "https://debyltech.com",
+    "schedule_name": "review-outreach-dev",
+    "schedule_group": "default"
+  },
+  "recovery": {
+    "schedule_name": "cart-recovery-dev",
+    "schedule_group": "default"
+  }
+}

ansible/roles/podman/templates/fulfillr/production.json.j2

@@ -1,15 +1,23 @@
+{# Production back-office config (fulfillr). Live tier:
+   - ecommerce store -> PROD Turso (fulfillr_prod_store_*)
+   - EasyPost + Stripe -> LIVE keys (fulfillr_prod_easypost_api_key / fulfillr_prod_stripe_api_key)
+   - AWS -> Fulfillr prod key (fulfillr_prod_access_key/secret_key), prod DynamoDB tables +
+     debyltech.digital.prod. fulfillr_region, fulfillr_tax_ein and fulfillr_hmac_arn are
+     shared vars (no dev/prod split). Mirrors dev.json.j2. (Snipcart removed post-migration.) #}
 {
-  "snipcart_api_key": "{{ snipcart_api_key }}",
   "easypost_api_key": "{{ easypost_api_key }}",
-  "stripe_api_key": "{{ stripe_api_key }}",
-  "backinstock_table": "{{ fulfillr_backinstock_table }}",
-  "cases_table": "{{ fulfillr_cases_table }}",
-  "tickets_table": "{{ fulfillr_tickets_table }}",
+  "stripe_api_key": "{{ fulfillr_prod_stripe_api_key }}",
+  "backinstock_table": "debyltech-backinstock-prod",
+  "cases_table": "debyltech-cases-prod",
+  "tickets_table": "debyltech-tickets-prod",
+  "store_database_url": "{{ fulfillr_prod_store_database_url }}",
+  "store_auth_token": "{{ fulfillr_prod_store_auth_token }}",
+  "download_base_url": "https://api.debyltech.com",
   "aws": {
-    "access_key": "{{ fulfillr_access_key }}",
-    "secret_key": "{{ fulfillr_secret_key }}",
+    "access_key": "{{ fulfillr_prod_access_key }}",
+    "secret_key": "{{ fulfillr_prod_secret_key }}",
     "region": "{{ fulfillr_region }}",
-    "bucket": "{{ fulfillr_bucket }}"
+    "bucket": "debyltech.digital.prod"
   },
   "tax": {
     "ein": "{{ fulfillr_tax_ein }}",
@@ -35,6 +43,12 @@
     "ses_from_email": "noreply@debyltech.com",
     "ses_reply_to": "support@debyltech.com",
     "ses_region": "us-east-1",
-    "base_url": "https://debyltech.com"
+    "base_url": "https://debyltech.com",
+    "schedule_name": "review-outreach-prod",
+    "schedule_group": "default"
+  },
+  "recovery": {
+    "schedule_name": "cart-recovery-prod",
+    "schedule_group": "default"
   }
 }

ansible/roles/podman/templates/searxng/settings.yml.j2

@@ -1,35 +0,0 @@
-use_default_settings: true
-
-general:
-  instance_name: "SearXNG"
-  debug: false
-
-server:
-  bind_address: "127.0.0.1"
-  port: 8080
-  secret_key: "{{ searxng_secret_key }}"
-  limiter: false
-  image_proxy: false
-
-search:
-  safe_search: 0
-  formats:
-    - html
-    - json
-
-engines:
-  - name: duckduckgo
-    engine: duckduckgo
-    disabled: false
-
-  - name: google
-    engine: google
-    disabled: false
-
-  - name: wikipedia
-    engine: wikipedia
-    disabled: false
-
-  - name: bing
-    engine: bing
-    disabled: false