Compare commits
24 Commits
chore/git-
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e5347cb946 | |||
| 9d3b695f9b | |||
| 670db289b1 | |||
| 1221bb5ead | |||
| de0af51d42 | |||
| ffff650e3a | |||
| 40eb38441f | |||
| 1b6616e937 | |||
| f7be98d2f7 | |||
| eed1fe2c4e | |||
| 9af1b171d9 | |||
| 1dd4b0adda | |||
| 086d1e89e2 | |||
| 7b8b7c469a | |||
| 7fe78a06e9 | |||
| 33e7440342 | |||
| fa05731d00 | |||
| 84b6911960 | |||
| 5087f47958 | |||
| 7d7fe9b364 | |||
|
|
eba689a93e | ||
| ec7ba4d9b0 | |||
| f15aa44e9f | |||
| 8b23ecbe69 |
8
.gitignore
vendored
8
.gitignore
vendored
@ -15,3 +15,11 @@ litellm/.infra-litellm-state
|
||||
*.snapshot
|
||||
*/secrets/
|
||||
git-bridge/state/
|
||||
|
||||
litellm/.env
|
||||
mem0/.env
|
||||
mem0/history/
|
||||
python_worker/app/
|
||||
python_worker/runtime.env
|
||||
python_worker/.migration-state
|
||||
python_worker/.staged-from-n8n
|
||||
|
||||
@ -46,11 +46,12 @@ services:
|
||||
GITEA__database__USER: gitea
|
||||
GITEA__database__PASSWD__FILE: /run/secrets/gitea_db_password
|
||||
GITEA__database__SSL_MODE: disable
|
||||
GITEA__server__DOMAIN: localhost
|
||||
GITEA__server__ROOT_URL: http://localhost:13000/
|
||||
GITEA__server__SSH_DOMAIN: localhost
|
||||
GITEA__server__SSH_PORT: 2223
|
||||
GITEA__service__REQUIRE_SIGNIN_VIEW: "true"
|
||||
GITEA__server__DOMAIN: gitea.xyslh.xyz
|
||||
GITEA__server__ROOT_URL: https://gitea.xyslh.xyz/
|
||||
GITEA__server__SSH_DOMAIN: gitea.xyslh.xyz
|
||||
GITEA__server__SSH_PORT: 2222
|
||||
GITEA__server__PROTOCOL: http
|
||||
GITEA__service__REQUIRE_SIGNIN_VIEW: "false"
|
||||
GITEA__service__DISABLE_REGISTRATION: "true"
|
||||
GITEA__security__INSTALL_LOCK: "true"
|
||||
depends_on:
|
||||
@ -63,8 +64,8 @@ services:
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
ports:
|
||||
- "127.0.0.1:13000:3000"
|
||||
- "127.0.0.1:2223:22"
|
||||
- "13000:3000"
|
||||
- "2223:22"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "grep -Eq '^INSTALL_LOCK[[:space:]]*=[[:space:]]*true' /data/gitea/conf/app.ini && grep -Eq '^DB_TYPE[[:space:]]*=[[:space:]]*postgres' /data/gitea/conf/app.ini && wget -q --spider http://127.0.0.1:3000/api/healthz"]
|
||||
interval: 5s
|
||||
@ -75,6 +76,8 @@ services:
|
||||
- git_forge
|
||||
|
||||
litellm:
|
||||
env_file:
|
||||
- ./litellm/.env
|
||||
container_name: litellm
|
||||
image: ghcr.io/berriai/litellm:main-latest
|
||||
restart: unless-stopped
|
||||
@ -145,6 +148,86 @@ services:
|
||||
networks:
|
||||
- git_forge
|
||||
|
||||
mem0:
|
||||
container_name: mem0
|
||||
build:
|
||||
context: ./mem0
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
MEM0_SOURCE_REF: 42cf18c4e6adb448e981aa1c7b55c1602b0cb670
|
||||
MEM0AI_VERSION: 2.0.12
|
||||
HTTP_PROXY: ${MEM0_BUILD_PROXY:-http://192.168.0.4:7890}
|
||||
HTTPS_PROXY: ${MEM0_BUILD_PROXY:-http://192.168.0.4:7890}
|
||||
NO_PROXY: localhost,127.0.0.1,postgres,litellm
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
env_file:
|
||||
- ./mem0/.env
|
||||
environment:
|
||||
POSTGRES_HOST: postgres
|
||||
POSTGRES_PORT: "5432"
|
||||
POSTGRES_DB: mem0
|
||||
POSTGRES_USER: mem0
|
||||
POSTGRES_COLLECTION_NAME: mem0_bootstrap_unused
|
||||
APP_DB_NAME: mem0_app
|
||||
OPENAI_API_KEY: internal-litellm
|
||||
OPENAI_BASE_URL: http://host.docker.internal:4000/v1
|
||||
MEM0_DEFAULT_LLM_MODEL: grok-fast
|
||||
MEM0_DEFAULT_EMBEDDER_MODEL: voyage-4
|
||||
AUTH_DISABLED: "false"
|
||||
MEM0_TELEMETRY: "false"
|
||||
DASHBOARD_URL: http://127.0.0.1:8888
|
||||
HISTORY_DB_PATH: /app/history/history.db
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
volumes:
|
||||
- ./mem0/history:/app/history
|
||||
ports:
|
||||
- "127.0.0.1:8888:8000"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/openapi.json', timeout=3).read()"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 18
|
||||
start_period: 30s
|
||||
networks:
|
||||
- git_forge
|
||||
|
||||
python_worker:
|
||||
container_name: python_worker
|
||||
image: n8n-python_worker
|
||||
pull_policy: never
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 60s
|
||||
working_dir: /app
|
||||
env_file:
|
||||
- ./python_worker/runtime.env
|
||||
volumes:
|
||||
- ./python_worker/app:/app
|
||||
- /volume2/docker/n8n/files:/files
|
||||
ports:
|
||||
- "8000:8000"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -lc
|
||||
- . /app/.venv/bin/activate && exec python -m uvicorn fastapi_app:app --host 0.0.0.0 --port 8000 --log-level debug
|
||||
dns:
|
||||
- 223.5.5.5
|
||||
- 114.114.114.114
|
||||
healthcheck:
|
||||
test: ["CMD", "/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 18
|
||||
start_period: 30s
|
||||
networks:
|
||||
n8n_backbone:
|
||||
aliases:
|
||||
- python_worker
|
||||
|
||||
networks:
|
||||
git_forge:
|
||||
name: git_forge
|
||||
@ -152,6 +235,10 @@ networks:
|
||||
external: true
|
||||
name: n8n_default
|
||||
|
||||
n8n_backbone:
|
||||
external: true
|
||||
name: n8n_default
|
||||
|
||||
secrets:
|
||||
postgres_superuser_password:
|
||||
file: ./postgres/secrets/postgres_superuser_password
|
||||
|
||||
@ -11,3 +11,19 @@ docker compose --profile git-bridge run --rm --no-deps git-bridge
|
||||
|
||||
GitHub remains a development workspace. Production deployment continues to
|
||||
use only Gitea Main.
|
||||
|
||||
## GitHub branch import
|
||||
|
||||
Import one reviewed cloud-development branch at a time:
|
||||
|
||||
```bash
|
||||
sudo env GITHUB_BRANCH=codex/example \
|
||||
/volume2/docker/infra/git-bridge/bin/import-github-branch.sh
|
||||
```
|
||||
|
||||
Accepted prefixes are `codex/`, `grok/`, and `cloud/`. Before a normal push to
|
||||
the same Gitea branch and creation of a Gitea pull request, the importer checks
|
||||
that both Main branches match, the candidate descends from Main, an existing
|
||||
Gitea branch can fast-forward, the canonical Compose renders, shell scripts
|
||||
parse, and no known secret or runtime-data path is tracked. It never merges,
|
||||
deploys, force-pushes, deletes a branch, or writes Main.
|
||||
|
||||
311
git-bridge/bin/import-github-branch.sh
Executable file
311
git-bridge/bin/import-github-branch.sh
Executable file
@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
umask 077
|
||||
|
||||
branch=${GITHUB_BRANCH:-}
|
||||
infra_dir=${INFRA_DIR:-/volume2/docker/infra}
|
||||
proxy_url=${GITHUB_PROXY_URL:-http://192.168.0.4:7890}
|
||||
compose_file=$infra_dir/docker-compose.yaml
|
||||
state_dir=$infra_dir/git-bridge/state
|
||||
state_repo=$state_dir/import.git
|
||||
candidate=$state_dir/import-candidate
|
||||
gitea_token_file=$infra_dir/git-bridge/secrets/gitea_token
|
||||
github_token_file=$infra_dir/git-bridge/secrets/github_token
|
||||
api_base=http://127.0.0.1:13000/api/v1
|
||||
temp_dir=
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "refusing import: run with sudo" >&2
|
||||
exit 2
|
||||
fi
|
||||
case "$branch" in
|
||||
codex/*|grok/*|cloud/*) ;;
|
||||
*)
|
||||
echo "refusing import: GITHUB_BRANCH must use codex/, grok/, or cloud/ prefix" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$branch" in
|
||||
*[!A-Za-z0-9._/-]*|*..*|*/.|/*|*/|*//*|main)
|
||||
echo "refusing import: unsafe branch name" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
for command_name in docker curl python3; do
|
||||
command -v "$command_name" >/dev/null 2>&1 || {
|
||||
echo "$command_name is required" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
docker info >/dev/null
|
||||
[ -f "$compose_file" ] || {
|
||||
echo "canonical Infra Compose is missing" >&2
|
||||
exit 2
|
||||
}
|
||||
[ ! -e "$infra_dir/compose.yaml" ] || {
|
||||
echo "legacy compose.yaml must not exist" >&2
|
||||
exit 2
|
||||
}
|
||||
for secret_file in "$gitea_token_file" "$github_token_file"; do
|
||||
[ -s "$secret_file" ] || {
|
||||
echo "bridge secret is missing: $secret_file" >&2
|
||||
exit 2
|
||||
}
|
||||
[ "$(stat -c '%a' "$secret_file")" = "600" ] || {
|
||||
echo "bridge secret permissions must be 0600: $secret_file" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
[ ! -e "$candidate" ] || {
|
||||
echo "stale candidate worktree exists: $candidate" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
temp_dir=$(mktemp -d /tmp/infra-branch-import.XXXXXX)
|
||||
askpass_file=$temp_dir/askpass.sh
|
||||
curl_config=$temp_dir/gitea-curl.conf
|
||||
response_file=$temp_dir/response.json
|
||||
request_file=$temp_dir/request.json
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
if [ -d "$candidate" ]; then
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git worktree remove --force \
|
||||
/infra/git-bridge/state/import-candidate >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [ -n "$temp_dir" ]; then
|
||||
rm -f "$askpass_file" "$curl_config" "$response_file" "$request_file" \
|
||||
"$temp_dir/tracked.zlist"
|
||||
rmdir "$temp_dir" 2>/dev/null || true
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
|
||||
cat >"$askpass_file" <<'ASKPASS'
|
||||
#!/bin/sh
|
||||
case "$1" in
|
||||
*Username*github.com*|*github.com*Username*) printf '%s\n' 'RX-Penguin' ;;
|
||||
*github.com*) cat /run/secrets/github_token ;;
|
||||
*Username*gitea*|*gitea*Username*) printf '%s\n' 'git-bridge' ;;
|
||||
*gitea*) cat /run/secrets/gitea_token ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
ASKPASS
|
||||
chmod 0500 "$askpass_file"
|
||||
|
||||
gitea_token=$(cat "$gitea_token_file")
|
||||
printf 'header = "Authorization: token %s"\n' "$gitea_token" >"$curl_config"
|
||||
chmod 0600 "$curl_config"
|
||||
gitea_token=
|
||||
|
||||
gitea_image=$(docker container inspect --format '{{.Config.Image}}' gitea)
|
||||
git_run() {
|
||||
docker run --rm \
|
||||
--user 0:0 \
|
||||
--network git_forge \
|
||||
--entrypoint git \
|
||||
-e GIT_TERMINAL_PROMPT=0 \
|
||||
-e GIT_ASKPASS=/run/secrets/askpass.sh \
|
||||
-e HTTP_PROXY="$proxy_url" \
|
||||
-e HTTPS_PROXY="$proxy_url" \
|
||||
-e ALL_PROXY="$proxy_url" \
|
||||
-e NO_PROXY=localhost,127.0.0.1,::1,gitea,postgres,litellm \
|
||||
-v "$infra_dir:/infra" \
|
||||
-v "$gitea_token_file:/run/secrets/gitea_token:ro" \
|
||||
-v "$github_token_file:/run/secrets/github_token:ro" \
|
||||
-v "$askpass_file:/run/secrets/askpass.sh:ro" \
|
||||
"$gitea_image" \
|
||||
-c safe.directory='*' "$@"
|
||||
}
|
||||
|
||||
git_run check-ref-format --branch "$branch" >/dev/null
|
||||
install -d -m 0700 "$state_dir"
|
||||
if [ ! -d "$state_repo" ]; then
|
||||
git_run init --bare /infra/git-bridge/state/import.git >/dev/null
|
||||
fi
|
||||
|
||||
if git_run --git-dir=/infra/git-bridge/state/import.git remote get-url gitea >/dev/null 2>&1; then
|
||||
[ "$(git_run --git-dir=/infra/git-bridge/state/import.git remote get-url gitea)" = \
|
||||
"http://gitea:3000/slh/infra.git" ] || {
|
||||
echo "gitea remote mismatch" >&2
|
||||
exit 2
|
||||
}
|
||||
else
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git remote add gitea \
|
||||
http://gitea:3000/slh/infra.git
|
||||
fi
|
||||
if git_run --git-dir=/infra/git-bridge/state/import.git remote get-url github >/dev/null 2>&1; then
|
||||
[ "$(git_run --git-dir=/infra/git-bridge/state/import.git remote get-url github)" = \
|
||||
"https://github.com/RX-Penguin/infra.git" ] || {
|
||||
echo "github remote mismatch" >&2
|
||||
exit 2
|
||||
}
|
||||
else
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git remote add github \
|
||||
https://github.com/RX-Penguin/infra.git
|
||||
fi
|
||||
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git fetch --no-tags gitea \
|
||||
'+refs/heads/main:refs/remotes/gitea/main'
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git fetch --no-tags github \
|
||||
"+refs/heads/main:refs/remotes/github/main" \
|
||||
"+refs/heads/$branch:refs/remotes/github/$branch"
|
||||
gitea_main=$(git_run --git-dir=/infra/git-bridge/state/import.git rev-parse refs/remotes/gitea/main)
|
||||
github_main=$(git_run --git-dir=/infra/git-bridge/state/import.git rev-parse refs/remotes/github/main)
|
||||
candidate_sha=$(git_run --git-dir=/infra/git-bridge/state/import.git rev-parse "refs/remotes/github/$branch")
|
||||
|
||||
[ "$gitea_main" = "$github_main" ] || {
|
||||
echo "import refused: GitHub Main differs from Gitea Main; run forward sync first" >&2
|
||||
exit 42
|
||||
}
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git merge-base --is-ancestor \
|
||||
"$gitea_main" "$candidate_sha" || {
|
||||
echo "import refused: candidate branch does not descend from current Gitea Main" >&2
|
||||
exit 42
|
||||
}
|
||||
[ "$candidate_sha" != "$gitea_main" ] || {
|
||||
echo "import refused: candidate branch contains no change from Gitea Main" >&2
|
||||
exit 42
|
||||
}
|
||||
|
||||
if git_run --git-dir=/infra/git-bridge/state/import.git ls-remote --exit-code \
|
||||
--heads gitea "refs/heads/$branch" >/dev/null 2>&1; then
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git fetch --no-tags gitea \
|
||||
"+refs/heads/$branch:refs/remotes/gitea/$branch"
|
||||
current_sha=$(git_run --git-dir=/infra/git-bridge/state/import.git \
|
||||
rev-parse "refs/remotes/gitea/$branch")
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git merge-base --is-ancestor \
|
||||
"$current_sha" "$candidate_sha" || {
|
||||
echo "import refused: existing Gitea branch cannot fast-forward" >&2
|
||||
exit 42
|
||||
}
|
||||
fi
|
||||
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git worktree add --detach \
|
||||
/infra/git-bridge/state/import-candidate "$candidate_sha" >/dev/null
|
||||
|
||||
[ -f "$candidate/docker-compose.yaml" ] || {
|
||||
echo "candidate test failed: docker-compose.yaml is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
[ ! -e "$candidate/compose.yaml" ] || {
|
||||
echo "candidate test failed: a second Compose entrypoint exists" >&2
|
||||
exit 1
|
||||
}
|
||||
docker compose -f "$candidate/docker-compose.yaml" config -q
|
||||
find "$candidate" -type f -name '*.sh' -exec sh -n '{}' ';'
|
||||
|
||||
git_run -C /infra/git-bridge/state/import-candidate ls-files -z >"$temp_dir/tracked.zlist"
|
||||
python3 - "$temp_dir/tracked.zlist" <<'PY'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
items = pathlib.Path(sys.argv[1]).read_bytes().split(b"\0")
|
||||
for raw in items:
|
||||
if not raw:
|
||||
continue
|
||||
path = raw.decode("utf-8", "strict")
|
||||
parts = path.split("/")
|
||||
unsafe = (
|
||||
path == ".env"
|
||||
or path.endswith("/.env")
|
||||
or "secrets" in parts
|
||||
or path.endswith("auth.json")
|
||||
or path.startswith("git-bridge/state/")
|
||||
or path.startswith("gitea/")
|
||||
or path.startswith("postgres/data/")
|
||||
or path.startswith("litellm/chatgpt/")
|
||||
or path.startswith("litellm/xai_oauth/")
|
||||
)
|
||||
if unsafe:
|
||||
raise SystemExit(f"candidate test failed: runtime or secret path is tracked: {path}")
|
||||
PY
|
||||
|
||||
# This is deliberately a normal push. Existing branches must fast-forward;
|
||||
# Git rejects divergence and the protected Main branch is never a destination.
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git push gitea \
|
||||
"refs/remotes/github/$branch:refs/heads/$branch"
|
||||
verified_sha=$(git_run --git-dir=/infra/git-bridge/state/import.git ls-remote gitea \
|
||||
"refs/heads/$branch" | awk 'NR == 1 { print $1 }')
|
||||
[ "$verified_sha" = "$candidate_sha" ] || {
|
||||
echo "import verification failed" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
status=$(curl -sS --config "$curl_config" -o "$response_file" -w '%{http_code}' \
|
||||
--get \
|
||||
--data-urlencode 'state=open' \
|
||||
--data-urlencode 'base=main' \
|
||||
--data-urlencode "head=$branch" \
|
||||
"$api_base/repos/slh/infra/pulls")
|
||||
[ "$status" = "200" ] || {
|
||||
echo "failed to inspect existing Gitea pull requests (HTTP $status)" >&2
|
||||
exit 1
|
||||
}
|
||||
pr_url=$(python3 - "$response_file" "$branch" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
for item in json.load(open(sys.argv[1], encoding="utf-8")):
|
||||
head = item.get("head") or {}
|
||||
if head.get("ref") == sys.argv[2]:
|
||||
print(item.get("html_url") or "")
|
||||
break
|
||||
PY
|
||||
)
|
||||
|
||||
if [ -z "$pr_url" ]; then
|
||||
python3 - "$request_file" "$branch" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
branch = sys.argv[2]
|
||||
payload = {
|
||||
"base": "main",
|
||||
"head": branch,
|
||||
"title": f"Import GitHub branch: {branch}",
|
||||
"body": (
|
||||
"Imported by git-bridge after branch ancestry, canonical Compose, "
|
||||
"Compose render, shell syntax, and tracked-secret boundary checks. "
|
||||
"No automatic merge or deployment was performed."
|
||||
),
|
||||
}
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
PY
|
||||
status=$(curl -sS --config "$curl_config" -o "$response_file" -w '%{http_code}' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-X POST "$api_base/repos/slh/infra/pulls" \
|
||||
--data-binary "@$request_file")
|
||||
[ "$status" = "201" ] || {
|
||||
echo "branch imported but Gitea PR creation failed (HTTP $status)" >&2
|
||||
exit 1
|
||||
}
|
||||
pr_url=$(python3 -c \
|
||||
'import json,sys; print(json.load(open(sys.argv[1])).get("html_url", ""))' \
|
||||
"$response_file")
|
||||
fi
|
||||
[ -n "$pr_url" ] || {
|
||||
echo "Gitea PR URL is unavailable" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
git_run --git-dir=/infra/git-bridge/state/import.git worktree remove --force \
|
||||
/infra/git-bridge/state/import-candidate >/dev/null
|
||||
rm -f "$askpass_file" "$curl_config" "$response_file" "$request_file" \
|
||||
"$temp_dir/tracked.zlist"
|
||||
rmdir "$temp_dir"
|
||||
temp_dir=
|
||||
trap - EXIT INT TERM
|
||||
|
||||
echo "GitHub branch import completed"
|
||||
echo "branch=$branch"
|
||||
echo "commit=$candidate_sha"
|
||||
echo "gitea_pull_request=$pr_url"
|
||||
echo "tests=passed"
|
||||
echo "force_push=false"
|
||||
echo "main_modified=false"
|
||||
echo "production_started=false"
|
||||
5
git-bridge/tests/bridge-smoke.md
Normal file
5
git-bridge/tests/bridge-smoke.md
Normal file
@ -0,0 +1,5 @@
|
||||
# Git bridge smoke test
|
||||
|
||||
This file is a harmless round-trip marker created on the GitHub development
|
||||
workspace. It must return through the tested Gitea branch-import workflow and
|
||||
a human-reviewed Gitea pull request before it can enter the source of truth.
|
||||
@ -72,6 +72,15 @@ model_list:
|
||||
model: xai/grok-build-0.1
|
||||
use_xai_oauth: true
|
||||
|
||||
- model_name: voyage-4
|
||||
model_info:
|
||||
mode: embedding
|
||||
context_window: 32000
|
||||
litellm_params:
|
||||
model: voyage/voyage-4
|
||||
api_key: os.environ/VOYAGE_API_KEY
|
||||
additional_drop_params: ["encoding_format"]
|
||||
|
||||
litellm_settings:
|
||||
aiohttp_trust_env: true
|
||||
drop_params: true
|
||||
|
||||
30
mem0/Dockerfile
Normal file
30
mem0/Dockerfile
Normal file
@ -0,0 +1,30 @@
|
||||
FROM python:3.12-slim-bookworm AS source
|
||||
|
||||
ARG MEM0_SOURCE_REF=42cf18c4e6adb448e981aa1c7b55c1602b0cb670
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /src
|
||||
RUN git init \
|
||||
&& git remote add origin https://github.com/mem0ai/mem0.git \
|
||||
&& git fetch --depth 1 origin "${MEM0_SOURCE_REF}" \
|
||||
&& test "$(git rev-parse FETCH_HEAD)" = "${MEM0_SOURCE_REF}" \
|
||||
&& git checkout --detach FETCH_HEAD
|
||||
|
||||
FROM python:3.12-slim-bookworm
|
||||
|
||||
ARG MEM0AI_VERSION=2.0.12
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
WORKDIR /app
|
||||
COPY --from=source /src/server/requirements.txt /tmp/requirements.txt
|
||||
RUN sed -i -E "s/^mem0ai.*/mem0ai==${MEM0AI_VERSION}/" /tmp/requirements.txt \
|
||||
&& pip install --no-cache-dir -r /tmp/requirements.txt \
|
||||
&& rm -f /tmp/requirements.txt
|
||||
COPY --from=source /src/server/ /app/
|
||||
RUN mkdir -p /app/history
|
||||
EXPOSE 8000
|
||||
CMD ["sh", "-c", "alembic upgrade head && exec uvicorn main:app --host 0.0.0.0 --port 8000"]
|
||||
125
mem0/README.md
Normal file
125
mem0/README.md
Normal file
@ -0,0 +1,125 @@
|
||||
# Mem0 记忆服务
|
||||
|
||||
Mem0 是 Infra Compose 管理的长期记忆服务。它只在宿主机
|
||||
`127.0.0.1:8888` 暴露 REST API,并启用 API Key 鉴权;不部署 Dashboard、
|
||||
Neo4j 或独立 Qdrant。
|
||||
|
||||
## 当前架构
|
||||
|
||||
- 容器:`mem0`
|
||||
- Compose 真相源:`/volume2/docker/infra/docker-compose.yaml`
|
||||
- API:`http://127.0.0.1:8888`
|
||||
- 关系数据库:Infra PostgreSQL 中独立的 `mem0_app` 数据库
|
||||
- 向量数据:Infra PostgreSQL 中启用 pgvector 的 `mem0` 数据库
|
||||
- 记忆抽取模型:LiteLLM 的 `grok-fast`
|
||||
- Embedding:LiteLLM 的 `voyage-4`,固定 1024 维
|
||||
- LiteLLM 地址:容器内通过 `http://host.docker.internal:4000/v1` 访问
|
||||
|
||||
Mem0 不持有 Grok、Voyage 或 OpenAI 的供应商凭据。模型凭据只归 LiteLLM
|
||||
管理,Mem0 使用内部占位 Key 访问本机网关。
|
||||
|
||||
## 宿主机目录
|
||||
|
||||
- `mem0/Dockerfile`:固定 Mem0 OSS `v2.0.12` 源码提交并构建服务镜像
|
||||
- `mem0/.env`:仅保存三个运行 Secret,不进入 Git,权限必须为 `0600`
|
||||
- `mem0/history/`:服务的本地历史状态,不进入 Git
|
||||
- PostgreSQL 数据:由 `postgres` 服务的数据目录统一持久化
|
||||
|
||||
`mem0/.env` 只允许以下键:
|
||||
|
||||
```text
|
||||
POSTGRES_PASSWORD
|
||||
JWT_SECRET
|
||||
ADMIN_API_KEY
|
||||
```
|
||||
|
||||
禁止将 `.env`、数据库文件、历史目录、模型 Key 或响应内容提交到仓库。
|
||||
|
||||
## 兼容性修复
|
||||
|
||||
最终运行镜像必须安装 Debian `libpq5`。`psycopg` 默认可退回纯 Python
|
||||
实现,但仍需系统 `libpq`;缺失时 Alembic 会在容器启动阶段报
|
||||
`no pq wrapper available`。
|
||||
|
||||
Mem0 使用 OpenAI SDK 调用 embedding 时会发送
|
||||
`encoding_format=\"float\"`,而 Voyage 仅接受 `base64`。因此 LiteLLM 的
|
||||
`voyage-4` 路由必须配置:
|
||||
|
||||
```yaml
|
||||
additional_drop_params: ["encoding_format"]
|
||||
```
|
||||
|
||||
该规则只作用于 `voyage-4`,不得改成全局丢弃参数,避免影响 Grok、GPT 和
|
||||
其他模型。
|
||||
|
||||
## 部署与完整测试
|
||||
|
||||
合并 Gitea PR 并完成 FF-only 同步后,执行:
|
||||
|
||||
```bash
|
||||
cd /volume2/docker
|
||||
sudo env INFRA_MEM0_DEPLOY=YES \
|
||||
./ops/deploy-and-test-infra-mem0.sh
|
||||
```
|
||||
|
||||
脚本会:
|
||||
|
||||
1. 校验 Secret 文件权限与键集合。
|
||||
2. 幂等创建或更新 `mem0` 数据库角色以及 `mem0`、`mem0_app` 数据库。
|
||||
3. 确认 pgvector 扩展存在。
|
||||
4. 构建镜像并先验证 `psycopg` 能加载 pq 实现。
|
||||
5. 仅重建 `mem0` 容器。
|
||||
6. 配置 `grok-fast`、`voyage-4` 和 1024 维向量存储。
|
||||
7. 创建一条临时记忆、检索验证后删除。
|
||||
|
||||
成功标志:
|
||||
|
||||
```text
|
||||
memory_create_search_delete_smoke=passed
|
||||
litellm_restarted=false
|
||||
other_services_restarted=none
|
||||
```
|
||||
|
||||
## 日常检查
|
||||
|
||||
```bash
|
||||
sudo docker compose \
|
||||
-f /volume2/docker/infra/docker-compose.yaml ps mem0
|
||||
|
||||
sudo docker inspect mem0 \
|
||||
--format 'status={{.State.Status}} restart={{.RestartCount}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'
|
||||
|
||||
curl -fsS --max-time 10 \
|
||||
http://127.0.0.1:8888/openapi.json >/dev/null
|
||||
```
|
||||
|
||||
API 业务请求必须携带 `X-API-Key`。检查时不得把 `ADMIN_API_KEY` 打印到终端、
|
||||
命令历史或日志。
|
||||
|
||||
## 故障排查
|
||||
|
||||
模型链路异常时运行只读诊断:
|
||||
|
||||
```bash
|
||||
cd /volume2/docker
|
||||
sudo ./ops/diagnose-infra-mem0-provider.sh
|
||||
```
|
||||
|
||||
诊断会从 Mem0 容器内分别测试 `grok-fast` 和 `voyage-4`,并输出脱敏后的
|
||||
Mem0/LiteLLM 错误。常见故障:
|
||||
|
||||
- `no pq wrapper available`:镜像缺少 `libpq5`,检查 Dockerfile 与实际镜像。
|
||||
- Voyage 拒绝 `encoding_format=float`:检查 LiteLLM 的单模型
|
||||
`additional_drop_params` 配置是否已部署。
|
||||
- HTTP 502 `provider_unavailable`:先直测 LiteLLM 对话与 embedding 路由,
|
||||
不要直接修改数据库。
|
||||
- 维度不一致:`voyage-4`、Mem0 embedder 和 pgvector collection 必须全部为
|
||||
1024 维;不要在已有 collection 上直接改维度。
|
||||
|
||||
## 安全与变更边界
|
||||
|
||||
- 不直接编辑 PostgreSQL 数据目录。
|
||||
- 不删除 `mem0`、`mem0_app` 数据库或 `mem0/history/` 来处理应用错误。
|
||||
- 不把 Mem0 端口暴露到局域网或公网;跨服务访问应走受控网络或反向代理。
|
||||
- 模型、维度、数据库结构和鉴权策略变更必须单独提 PR 并完成端到端测试。
|
||||
- 失败时保留容器日志、镜像和数据库现场,先诊断,再决定回滚方案。
|
||||
28
python_worker/README.md
Normal file
28
python_worker/README.md
Normal file
@ -0,0 +1,28 @@
|
||||
# Python Worker
|
||||
|
||||
`python_worker` 是 n8n、OGame Agent 与其他自动化流程共同使用的 FastAPI
|
||||
服务。本次迁移只调整运行归属和宿主机目录,不重构业务接口。
|
||||
|
||||
## 目录边界
|
||||
|
||||
- `app/`:从 `/volume2/docker/n8n/python_worker` 分两阶段同步的应用、虚拟
|
||||
环境和运行状态,不纳入 Git。
|
||||
- `runtime.env`:从旧容器提取的宿主机运行参数,权限必须为 `0600`,不包含
|
||||
模型或平台凭据,不纳入 Git。
|
||||
- `.migration-state`:切换成功后写入的回滚定位信息,不纳入 Git。
|
||||
- `README.md`:唯一纳入 Gitea 的目录说明。
|
||||
|
||||
容器继续使用 `python_worker` 名称、8000 端口和 `n8n_default` 网络,因此现有
|
||||
n8n 工作流及 OGame Agent 仍通过 `http://python_worker:8000` 访问,不需要修改
|
||||
调用地址。`/files` 仍是 n8n 与 Worker 共用的数据目录,本阶段不迁移。
|
||||
|
||||
## 迁移策略
|
||||
|
||||
迁移前先在旧 Worker 在线时把约 921 MB 的目录预复制到
|
||||
`/volume2/docker/infra/python_worker/app`。正式切换时停止旧容器,再执行一次
|
||||
增量同步,随后由 Infra Compose 启动新容器。旧目录和停止状态的旧容器都会
|
||||
保留,以便回滚。
|
||||
|
||||
当前沿用本地镜像 `n8n-python_worker` 和已有 `.venv`,这是降低首次迁移风险的
|
||||
过渡方案。后续必须单独补充固定版本的生产 Dockerfile,停止容器启动时联网安装
|
||||
依赖,再将该服务纳入可复现构建。
|
||||
Loading…
x
Reference in New Issue
Block a user