BudEcosystem · sonalibud · Jan 5, 2025 · Jan 6, 2025 · Jan 7, 2025 · Jan 7, 2025
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt
@@ -9,3 +9,5 @@ anthropic
 orjson==3.9.15
 pydantic==2.7.1
 google-cloud-aiplatform==1.43.0
+fastapi-sso==0.10.0
+uvloop==0.21.0
diff --git a/.dapr/appconfig-dev.yaml b/.dapr/appconfig-dev.yaml
@@ -0,0 +1,19 @@
+apiVersion: dapr.io/v1alpha1
+kind: Configuration
+metadata:
+  name: appconfig
+  namespace: default
+spec:
+  tracing:
+    samplingRate: "1"
+    stdout: true
+  features:
+    - name: SchedulerReminders
+      enabled: true
+  #    zipkin:
+  #      endpointAddress: http://localhost:9411/api/v2/spans
+  secrets:
+    scopes:
+      - storeName: secretstore-local
+        defaultAccess: allow
+        deniedSecrets: [ ]
diff --git a/.dapr/components/configstore.yaml b/.dapr/components/configstore.yaml
@@ -0,0 +1,20 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-configuration-stores/redis-configuration-store/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: configstore
+  namespace: development
+spec:
+  type: configuration.redis
+  metadata:
+    - name: redisHost
+      secretKeyRef:
+        name: REDIS_URI
+        key: REDIS_URI
+    - name: redisPassword
+      secretKeyRef:
+        name: REDIS_PASSWORD
+        key: REDIS_PASSWORD
+auth:
+  secretStore: secretstore-local
diff --git a/.dapr/components/pubsub-redis.yaml b/.dapr/components/pubsub-redis.yaml
@@ -0,0 +1,25 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-pubsub/setup-redis-pubsub/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: pubsub-redis
+  namespace: development
+spec:
+  type: pubsub.redis
+  version: v1
+  metadata:
+    - name: redisHost
+      secretKeyRef:
+        name: REDIS_URI
+        key: REDIS_URI
+    - name: redisPassword
+      secretKeyRef:
+        name: REDIS_PASSWORD
+        key: REDIS_PASSWORD
+    - name: consumerID
+      value: "{appID}"
+    - name: concurrency
+      value: "10"
+auth:
+  secretStore: secretstore-local
diff --git a/.dapr/components/secretstore-env.yaml b/.dapr/components/secretstore-env.yaml
@@ -0,0 +1,13 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-secret-stores/file-secret-store/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: secretstore-local
+  namespace: development
+spec:
+  type: secretstores.local.env
+  version: v1
+  metadata:
+    - name: prefix
+      value: "SECRETS_"
diff --git a/.dapr/components/statestore.yaml b/.dapr/components/statestore.yaml
@@ -0,0 +1,23 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-redis/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore
+  namespace: development
+spec:
+  type: state.redis
+  version: v1
+  metadata:
+    - name: redisHost
+      secretKeyRef:
+        name: REDIS_URI
+        key: REDIS_URI
+    - name: redisPassword
+      secretKeyRef:
+        name: REDIS_PASSWORD
+        key: REDIS_PASSWORD
+    - name: actorStateStore
+      value: "true"
+auth:
+  secretStore: secretstore-local
diff --git a/.dapr/components/subscriptions.yaml b/.dapr/components/subscriptions.yaml
@@ -0,0 +1,12 @@
+apiVersion: dapr.io/v2alpha1
+kind: Subscription
+metadata:
+  name: pubsub-subscription
+spec:
+  topic: budLitellmMessages
+  routes:
+    default: /notifications
+  pubsubname: pubsub-redis
+  deadLetterTopic: poisonMessages
+scopes:
+  - budlitellm
diff --git a/.dockerignore b/.dockerignore
@@ -9,3 +9,5 @@ tests
 .devcontainer
 *.tgz
 log.txt
+budlitellm
+docker/Dockerfile.*
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -22,7 +22,7 @@
 
 <!-- List of changes -->
 
-## [REQUIRED] Testing - Attach a screenshot of any new tests passing locall
+## [REQUIRED] Testing - Attach a screenshot of any new tests passing locally
 If UI changes, send a screenshot/GIF of working UI fixes
 
 <!-- Test procedure -->

diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py
@@ -1,6 +1,4 @@
-from locust import HttpUser, task, between, events
-import json
-import time
+from locust import HttpUser, task, between
 
 
 class MyUser(HttpUser):
@@ -10,7 +8,7 @@ class MyUser(HttpUser):
     def chat_completion(self):
         headers = {
             "Content-Type": "application/json",
-            "Authorization": f"Bearer sk-ZoHqrLIs2-5PzJrqBaviAA",
+            "Authorization": "Bearer sk-ZoHqrLIs2-5PzJrqBaviAA",
             # Include any additional headers you may need for authentication, etc.
         }
 

diff --git a/.github/workflows/reset_stable.yml b/.github/workflows/reset_stable.yml
@@ -0,0 +1,39 @@
+name: Reset litellm_stable branch
+
+on:
+  release:
+    types: [published, created]
+jobs:
+  update-stable-branch:
+    if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Reset litellm_stable_release_branch branch to the release commit
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Fetch all branches and tags
+          git fetch --all
+
+          # Check if the litellm_stable_release_branch branch exists
+          if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
+            echo "litellm_stable_release_branch branch exists."
+            git checkout litellm_stable_release_branch
+          else
+            echo "litellm_stable_release_branch branch does not exist. Creating it."
+            git checkout -b litellm_stable_release_branch
+          fi
+
+          # Reset litellm_stable_release_branch branch to the release commit
+          git reset --hard $GITHUB_SHA
+
+          # Push the updated litellm_stable_release_branch branch
+          git push origin litellm_stable_release_branch --force
diff --git a/.gitignore b/.gitignore
@@ -68,3 +68,7 @@ litellm_venv/*
 litellm/proxy/google-cloud-sdk/*
 budlitellm/*
 tests/llm_translation/log.txt
+venv/
+tests/local_testing/log.txt
+
+.codegpt
diff --git a/Dockerfile b/Dockerfile
@@ -1,18 +1,20 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
 # Set the working directory to /app
 WORKDIR /app
 
+USER root
+
 # Install build dependencies
-RUN apt-get clean && apt-get update && \
-    apt-get install -y gcc python3-dev && \
-    rm -rf /var/lib/apt/lists/*
+RUN apk update && \
+    apk add --no-cache gcc python3-dev openssl openssl-dev
+
 
 RUN pip install --upgrade pip && \
     pip install build
@@ -49,8 +51,12 @@ RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
-# Update dependencies and clean up - handles debian security issue
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* 
+# Ensure runtime stage runs as root
+USER root
+
+# Install runtime dependencies
+RUN apk update && \
+    apk add --no-cache openssl
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -67,10 +73,11 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # Generate prisma client
 RUN prisma generate
 RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs 
 CMD ["--port", "4000"]
diff --git a/Dockerfile.bud b/Dockerfile.bud
@@ -11,7 +11,7 @@ WORKDIR /app
 
 # Install build dependencies
 RUN apt-get clean && apt-get update && \
-    apt-get install -y gcc python3-dev && \
+    apt-get install -y gcc python3-dev git && \
     rm -rf /var/lib/apt/lists/*
 
 RUN pip install --upgrade pip && \
@@ -21,7 +21,7 @@ RUN pip install --upgrade pip && \
 COPY . .
 
 # Build Admin UI
-RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -44,13 +44,13 @@ RUN pip uninstall PyJWT -y
 RUN pip install PyJWT --no-cache-dir
 
 # Build Admin UI
-RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 # Update dependencies and clean up - handles debian security issue
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* 
+RUN apt-get update && apt-get install -y git && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* 
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -64,10 +64,14 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# ensure gptcache is installed
+RUN pip install git+https://github.com/BudEcosystem/BudServeGPTCache.git#egg=gptcache
+RUN pip install git+https://github.com/BudEcosystem/bud-microframe.git#egg=budmicroframe
+
 # Generate prisma client
 RUN prisma generate
-RUN chmod +x entrypoint.sh
+# RUN chmod +x entrypoint.sh
 
-EXPOSE 4000/tcp
+EXPOSE 4010/tcp
 
-ENTRYPOINT ["sh", "-c", "cp /app/config/config.yaml /app && litellm --config /app/config.yaml --port 4000"]
+# ENTRYPOINT ["sh", "-c", "litellm --config /app/litellm_config.yaml --port 4000"]
diff --git a/README.md b/README.md
@@ -175,12 +175,12 @@ for part in response:
 
 ## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
 
-LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack, MLflow
+LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
 
 ```python
 from litellm import completion
 
-## set env variables for logging tools
+## set env variables for logging tools (when using MLflow, no API key set up is required)
 os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
 os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
 os.environ["LANGFUSE_PUBLIC_KEY"] = ""
@@ -190,7 +190,7 @@ os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
 os.environ["OPENAI_API_KEY"]
 
 # set callbacks
-litellm.success_callback = ["lunary", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
+litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
 
 #openai call
 response = completion(model="anthropic/claude-3-sonnet-20240229", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
@@ -262,7 +262,7 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
 
 # Add the litellm salt key - you cannot change this after adding a model
 # It is used to encrypt / decrypt your LLM API Key credentials
-# We recommned - https://1password.com/password-generator/ 
+# We recommend - https://1password.com/password-generator/ 
 # password generator to get a random hash for litellm salt key
 echo 'LITELLM_SALT_KEY="sk-1234"' > .env
 
@@ -358,7 +358,7 @@ poetry install -E extra_proxy -E proxy
 Step 3: Test your change:
 
 ```
-cd litellm/tests # pwd: Documents/litellm/litellm/tests
+cd tests # pwd: Documents/litellm/litellm/tests
 poetry run flake8
 poetry run pytest .
 ```

diff --git a/cookbook/Benchmarking_LLMs_by_use_case.ipynb b/cookbook/Benchmarking_LLMs_by_use_case.ipynb