ci : full deploy with searxng, usgi, frontend (in progress)

docker-compose.yml

@@ -32,8 +32,8 @@ services:
       - ./searxng:/etc/searxng:rw
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
-      - UWSGI_WORKERS=4
-      - UWSGI_THREADS=4
+      - UWSGI_WORKERS=1
+      - UWSGI_THREADS=1
     cap_add:
       - CHOWN
       - SETGID
@@ -48,27 +48,6 @@ services:
     networks:
       - agentic-seek-net
 
-  backend:
-    container_name: backend
-    build:
-      context: .
-      dockerfile: Dockerfile.backend
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./.screenshots:/app/.screenshots:ro  # Mount screenshots folder
-      - ./app.py:/app/app.py:ro             # Mount backend code
-      - ./sources:/app/sources:ro           # Assuming sources/ contains your modules
-      - ./prompts:/app/prompts:ro           # Assuming prompts/ for agent prompts
-      - ./config.ini:/app/config.ini:ro     # Config file
-    environment:
-      - PYTHONUNBUFFERED=1
-    depends_on:
-      - searxng
-      - redis
-    networks:
-      - agentic-seek-net
-
   frontend:
     container_name: frontend
     build:
@@ -77,12 +56,11 @@ services:
     ports:
       - "3000:3000"
     volumes:
-      - ./frontend/agentic-seek/src:/app/src  # Live reload for development
+      - ./frontend/agentic-seek/src:/app/src
     environment:
       - NODE_ENV=development
       - CHOKIDAR_USEPOLLING=true  # Ensure file watching works in Docker
-    depends_on:
-      - backend
+      - BACKEND_URL=http://host.docker.internal:8000
     networks:
       - agentic-seek-net
 

frontend/agentic-seek/src/App.js

@@ -28,7 +28,7 @@ function App() {
 
         try {
             //const res = await axios.post('http://backend:8000/query', { ... });
-            const res = await axios.post('http://localhost:8000/query', {
+            const res = await axios.post('${process.env.BACKEND_URL}/query', {
                 query,
                 lang: 'en',
                 tts_enabled: false,
@@ -56,7 +56,7 @@ function App() {
     const handleGetScreenshot = async () => {
         try {
             //const res = await axios.get('http://backend:8000/screenshots/updated_screen.png');
-            const res = await axios.get('http://localhost:8000/screenshots/updated_screen.png');
+            const res = await axios.get('${process.env.BACKEND_URL}/screenshots/updated_screen.png');
             setResponseData((prev) => ({ ...prev, screenshot: res.data.screenshot }));
             setCurrentView('screenshot');
         } catch (err) {
@@ -153,7 +153,7 @@ function App() {
                                 </div>
                             ) : (
                                 <div className="screenshot">
-                                  <img src="http://localhost:8000/screenshots/updated_screen.png" alt="Screenshot" />
+                                  <img src="${process.env.BACKEND_URL}/screenshots/updated_screen.png" alt="Screenshot" />
                                 </div>
                             )}
                         </div>

searxng/docker-compose.yml

@@ -29,8 +29,8 @@ services:
       - ./searxng:/etc/searxng:rw
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
-      - UWSGI_WORKERS=4
-      - UWSGI_THREADS=4
+      - UWSGI_WORKERS=1
+      - UWSGI_THREADS=1
     cap_add:
       - CHOWN
       - SETGID

searxng/searxng/uwsgi.ini

@@ -5,11 +5,11 @@ gid = searxng
 
 # Number of workers (usually CPU count)
 # default value: %k (= number of CPU core, see Dockerfile)
-workers = 4
+workers = 1
 
 # Number of threads per worker
 # default value: 4 (see Dockerfile)
-threads = 4
+threads = 1
 
 # The right granted on the created socket
 chmod-socket = 666
@@ -19,7 +19,7 @@ single-interpreter = true
 master = true
 plugin = python3
 lazy-apps = true
-enable-threads = 4
+enable-threads = 1
 
 # Module to import
 module = searx.webapp
@@ -49,4 +49,4 @@ die-on-term
 # uwsgi serves the static files
 static-map = /static=/usr/local/searxng/searx/static
 static-gzip-all = True
-offload-threads = 4
+offload-threads = 1

searxng/setup_searxng.sh

@@ -1,72 +1,5 @@
 #!/bin/bash
 
-# Script to automate SearXNG setup and deployment with Docker Compose
-
-command_exists() {
-    command -v "$1" &> /dev/null
-}
-
-# Check if Docker is installed
-if ! command_exists docker; then
-    echo "Error: Docker is not installed. Please install Docker first."
-    echo "On Ubuntu: sudo apt install docker.io"
-    echo "On macOS/Windows: Install Docker Desktop from https://www.docker.com/get-started/"
-    exit 1
-fi
-
-# Check if Docker daemon is running
-echo "Checking if Docker daemon is running..."
-if ! docker info &> /dev/null; then
-    echo "Error: Docker daemon is not running or inaccessible."
-    if [ "$(uname)" = "Linux" ]; then
-        echo "Trying to start Docker service (may require sudo)..."
-        if sudo systemctl start docker &> /dev/null; then
-            echo "Docker started successfully."
-        else
-            echo "Failed to start Docker. Possible issues:"
-            echo "1. Run this script with sudo: sudo bash setup_searxng.sh"
-            echo "2. Check Docker installation: sudo systemctl status docker"
-            echo "3. Add your user to the docker group: sudo usermod -aG docker $USER (then log out and back in)"
-            exit 1
-        fi
-    else
-        echo "Please start Docker manually:"
-        echo "- On macOS/Windows: Open Docker Desktop."
-        echo "- On Linux: Run 'sudo systemctl start docker' or check your distro's docs."
-        exit 1
-    fi
-else
-    echo "Docker daemon is running."
-fi
-
-# Check if Docker Compose is installed
-if ! command_exists docker-compose; then
-    echo "Error: Docker Compose is not installed. Please install it first."
-    echo "On Ubuntu: sudo apt install docker-compose"
-    echo "Or via pip: pip install docker-compose"
-    exit 1
-fi
-
-# Create a directory for SearXNG config if it doesn’t exist
-mkdir -p searxng
-cd . || exit
-
-# Check if docker-compose.yml exists
-if [ ! -f "docker-compose.yml" ]; then
-    echo "Error: docker-compose.yml not found in the current directory."
-    echo "Please create it before running this script."
-    exit 1
-fi
-
-# Start containers to generate initial config files
-echo "Starting containers for initial setup..."
-if ! docker-compose up -d; then
-    echo "Error: Failed to start containers. Check Docker logs with 'docker compose logs'."
-    echo "Possible fixes: Run with sudo or ensure port 8080 is free."
-    exit 1
-fi
-sleep 10
-
 # Generate a secret key and update settings
 SECRET_KEY=$(openssl rand -hex 32)
 if [ -f "searxng/settings.yml" ]; then
@@ -83,9 +16,4 @@ else
     echo "Error: settings.yml not found. Initial setup may have failed."
     docker-compose logs searxng
     exit 1
-fi
-
-# Display status and access instructions
-echo "SearXNG setup complete!"
-docker ps -a --filter "name=searxng" --filter "name=redis"
-echo "Access SearXNG at: http://localhost:8080"
+fi

searxng/uwsgi.ini

@@ -0,0 +1,53 @@
+[uwsgi]
+# Who will run the code
+uid = searxng
+gid = searxng
+
+# Number of workers (usually CPU count)
+# default value: %k (= number of CPU core, see Dockerfile)
+workers = 1
+
+# Number of threads per worker
+# default value: 4 (see Dockerfile)
+enable-threads = false
+threads = 1
+
+# The right granted on the created socket
+chmod-socket = 666
+
+# Plugin to use and interpreter config
+single-interpreter = true
+master = true
+plugin = python3
+lazy-apps = true
+enable-threads = 1
+
+# Module to import
+module = searx.webapp
+
+# Virtualenv and python path
+pythonpath = /usr/local/searxng/
+chdir = /usr/local/searxng/searx/
+
+# automatically set processes name to something meaningful
+auto-procname = true
+
+# Disable request logging for privacy
+disable-logging = true
+log-5xx = true
+
+# Set the max size of a request (request-body excluded)
+buffer-size = 8192
+
+# No keep alive
+# See https://github.com/searx/searx-docker/issues/24
+add-header = Connection: close
+
+# Follow SIGTERM convention
+# See https://github.com/searxng/searxng/issues/3427
+die-on-term
+
+# uwsgi serves the static files
+static-map = /static=/usr/local/searxng/searx/static
+static-gzip-all = True
+offload-threads = 2

sources/browser.py

@@ -46,7 +46,7 @@ def get_chrome_path() -> str:
             return path
     return None
 
-def create_driver(headless=False, stealth_mode=True) -> webdriver.Chrome:
+def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx") -> webdriver.Chrome:
     """Create a Chrome WebDriver with specified options."""
     chrome_options = Options()
     chrome_path = get_chrome_path()
@@ -74,10 +74,10 @@ def create_driver(headless=False, stealth_mode=True) -> webdriver.Chrome:
     chrome_options.add_argument(f'--window-size={width},{height}')
     if not stealth_mode:
         # crx file can't be installed in stealth mode
-        crx_path = "./crx/nopecha.crx"
         if not os.path.exists(crx_path):
-            raise FileNotFoundError(f"Extension file not found at: {crx_path}")
-        chrome_options.add_extension(crx_path)
+            pretty_print(f"Anti-captcha CRX not found at {crx_path}.", color="failure")
+        else:
+            chrome_options.add_extension(crx_path)
     
     chromedriver_path = shutil.which("chromedriver")
     if not chromedriver_path:

start_services.sh

@@ -1,4 +1,73 @@
 #!/bin/bash
 
-# start searxng service for internet search 
-cd searxng && ./setup_searxng.sh
+command_exists() {
+    command -v "$1" &> /dev/null
+}
+
+#
+# Check if Docker is installed é running
+#
+
+if ! command_exists docker; then
+    echo "Error: Docker is not installed. Please install Docker first."
+    echo "On Ubuntu: sudo apt install docker.io"
+    echo "On macOS/Windows: Install Docker Desktop from https://www.docker.com/get-started/"
+    exit 1
+fi
+
+# Check if Docker daemon is running
+echo "Checking if Docker daemon is running..."
+if ! docker info &> /dev/null; then
+    echo "Error: Docker daemon is not running or inaccessible."
+    if [ "$(uname)" = "Linux" ]; then
+        echo "Trying to start Docker service (may require sudo)..."
+        if sudo systemctl start docker &> /dev/null; then
+            echo "Docker started successfully."
+        else
+            echo "Failed to start Docker. Possible issues:"
+            echo "1. Run this script with sudo: sudo bash setup_searxng.sh"
+            echo "2. Check Docker installation: sudo systemctl status docker"
+            echo "3. Add your user to the docker group: sudo usermod -aG docker $USER (then log out and back in)"
+            exit 1
+        fi
+    else
+        echo "Please start Docker manually:"
+        echo "- On macOS/Windows: Open Docker Desktop."
+        echo "- On Linux: Run 'sudo systemctl start docker' or check your distro's docs."
+        exit 1
+    fi
+else
+    echo "Docker daemon is running."
+fi
+
+# Check if Docker Compose is installed
+if ! command_exists docker-compose; then
+    echo "Error: Docker Compose is not installed. Please install it first."
+    echo "On Ubuntu: sudo apt install docker-compose"
+    echo "Or via pip: pip install docker-compose"
+    exit 1
+fi
+
+# Check if docker-compose.yml exists
+if [ ! -f "docker-compose.yml" ]; then
+    echo "Error: docker-compose.yml not found in the current directory."
+    exit 1
+fi
+
+# start searxng service for internet search service
+cd searxng && ./setup_searxng.sh && cd ..
+
+# start docker compose for searxng, redis, frontend services
+echo "SearXNG setup complete!"
+
+echo "Warning: stopping all docker containers (t-4 seconds)..."
+sleep 4
+docker stop $(docker ps -a -q)
+echo "All containers stopped"
+
+if ! docker-compose up; then
+    echo "Error: Failed to start containers. Check Docker logs with 'docker compose logs'."
+    echo "Possible fixes: Run with sudo or ensure port 8080 is free."
+    exit 1
+fi
+sleep 10