Initial commit

kustomize/components/shopping-assistant/scripts/1_deploy_alloydb_infra.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+# Replace me
+PROJECT_ID=$PROJECT_ID
+PROJECT_NUMBER=$PROJECT_NUMBER
+PGPASSWORD=$PGPASSWORD
+
+# Set sensible defaults
+REGION=us-central1
+USE_GKE_GCLOUD_AUTH_PLUGIN=True
+ALLOYDB_NETWORK=default
+ALLOYDB_SERVICE_NAME=onlineboutique-network-range
+ALLOYDB_CLUSTER_NAME=onlineboutique-cluster
+ALLOYDB_INSTANCE_NAME=onlineboutique-instance
+ALLOYDB_CARTS_DATABASE_NAME=carts
+ALLOYDB_CARTS_TABLE_NAME=cart_items
+ALLOYDB_PRODUCTS_DATABASE_NAME=products
+ALLOYDB_PRODUCTS_TABLE_NAME=catalog_items
+ALLOYDB_USER_GSA_NAME=alloydb-user-sa
+ALLOYDB_USER_GSA_ID=${ALLOYDB_USER_GSA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com
+ALLOYDB_SECRET_NAME=alloydb-secret
+
+# Enable services
+gcloud services enable alloydb.googleapis.com
+gcloud services enable servicenetworking.googleapis.com
+gcloud services enable secretmanager.googleapis.com
+gcloud services enable aiplatform.googleapis.com
+gcloud services enable generativelanguage.googleapis.com
+
+# Set our DB credentials behind the secret
+echo $PGPASSWORD | gcloud secrets create ${ALLOYDB_SECRET_NAME} --data-file=-
+
+# Set up needed service connection
+gcloud compute addresses create ${ALLOYDB_SERVICE_NAME} \
+    --global \
+    --purpose=VPC_PEERING \
+    --prefix-length=16 \
+    --description="Online Boutique Private Services" \
+    --network=${ALLOYDB_NETWORK}
+
+gcloud services vpc-peerings connect \
+    --service=servicenetworking.googleapis.com \
+    --ranges=${ALLOYDB_SERVICE_NAME} \
+    --network=${ALLOYDB_NETWORK}
+
+gcloud alloydb clusters create ${ALLOYDB_CLUSTER_NAME} \
+    --region=${REGION} \
+    --password=${PGPASSWORD} \
+    --disable-automated-backup \
+    --network=${ALLOYDB_NETWORK}
+
+gcloud alloydb instances create ${ALLOYDB_INSTANCE_NAME} \
+    --cluster=${ALLOYDB_CLUSTER_NAME} \
+    --region=${REGION} \
+    --cpu-count=4 \
+    --instance-type=PRIMARY
+
+gcloud alloydb instances create ${ALLOYDB_INSTANCE_NAME}-replica \
+    --cluster=${ALLOYDB_CLUSTER_NAME} \
+    --region=${REGION} \
+    --cpu-count=4 \
+    --instance-type=READ_POOL \
+    --read-pool-node-count=2
+
+gcloud beta alloydb instances update ${ALLOYDB_INSTANCE_NAME} \
+    --cluster=${ALLOYDB_CLUSTER_NAME} \
+    --region=${REGION} \
+    --assign-inbound-public-ip=ASSIGN_IPV4 \
+    --database-flags password.enforce_complexity=on
+
+# Fetch the primary and read IPs
+ALLOYDB_PRIMARY_IP=`gcloud alloydb instances list --region=${REGION} --cluster=${ALLOYDB_CLUSTER_NAME} --filter="INSTANCE_TYPE:PRIMARY" --format=flattened | sed -nE "s/ipAddress:\s*(.*)/\1/p"`
+ALLOYDB_READ_IP=`gcloud alloydb instances list --region=${REGION} --cluster=${ALLOYDB_CLUSTER_NAME} --filter="INSTANCE_TYPE:READ_POOL" --format=flattened | sed -nE "s/ipAddress:\s*(.*)/\1/p"`
+
+# Substitute environment values (alloydb/kustomization.yaml)
+sed -i "s/PROJECT_ID_VAL/${PROJECT_ID}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/REGION_VAL/${REGION}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_PRIMARY_IP_VAL/${ALLOYDB_PRIMARY_IP}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_USER_GSA_ID/${ALLOYDB_USER_GSA_ID}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_CLUSTER_NAME_VAL/${ALLOYDB_CLUSTER_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_INSTANCE_NAME_VAL/${ALLOYDB_INSTANCE_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_CARTS_DATABASE_NAME_VAL/${ALLOYDB_CARTS_DATABASE_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_CARTS_TABLE_NAME_VAL/${ALLOYDB_CARTS_TABLE_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_PRODUCTS_DATABASE_NAME_VAL/${ALLOYDB_PRODUCTS_DATABASE_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_PRODUCTS_TABLE_NAME_VAL/${ALLOYDB_PRODUCTS_TABLE_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+sed -i "s/ALLOYDB_SECRET_NAME_VAL/${ALLOYDB_SECRET_NAME}/g" kustomize/components/alloydb/kustomization.yaml
+
+# Substitute environment values (kustomize/components/shopping-assistant/shoppingassistantservice.yaml)
+sed -i "s/PROJECT_ID_VAL/${PROJECT_ID}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/REGION_VAL/${REGION}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/ALLOYDB_CLUSTER_NAME_VAL/${ALLOYDB_CLUSTER_NAME}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/ALLOYDB_INSTANCE_NAME_VAL/${ALLOYDB_INSTANCE_NAME}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/ALLOYDB_DATABASE_NAME_VAL/${ALLOYDB_PRODUCTS_DATABASE_NAME}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/ALLOYDB_TABLE_NAME_VAL/${ALLOYDB_PRODUCTS_TABLE_NAME}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/ALLOYDB_SECRET_NAME_VAL/${ALLOYDB_SECRET_NAME}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+sed -i "s/ALLOYDB_USER_GSA_ID/${ALLOYDB_USER_GSA_ID}/g" kustomize/components/shopping-assistant/shoppingassistantservice.yaml
+
+# Create service account for the cart and shopping assistant services
+gcloud iam service-accounts create ${ALLOYDB_USER_GSA_NAME} \
+    --display-name=${ALLOYDB_USER_GSA_NAME}
+
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member=serviceAccount:${ALLOYDB_USER_GSA_ID} --role=roles/alloydb.client
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member=serviceAccount:${ALLOYDB_USER_GSA_ID} --role=roles/alloydb.databaseUser
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member=serviceAccount:${ALLOYDB_USER_GSA_ID} --role=roles/secretmanager.secretAccessor
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member=serviceAccount:${ALLOYDB_USER_GSA_ID} --role=roles/serviceusage.serviceUsageConsumer
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member=serviceAccount:service-${PROJECT_NUMBER}@gcp-sa-alloydb.iam.gserviceaccount.com --role=roles/aiplatform.user
+
+gcloud iam service-accounts add-iam-policy-binding ${ALLOYDB_USER_GSA_ID} \
+    --member "serviceAccount:${PROJECT_ID}.svc.id.goog[default/cartservice]" \
+    --role roles/iam.workloadIdentityUser
+
+gcloud iam service-accounts add-iam-policy-binding ${ALLOYDB_USER_GSA_ID} \
+    --member "serviceAccount:${PROJECT_ID}.svc.id.goog[default/shoppingassistantservice]" \
+    --role roles/iam.workloadIdentityUser
+
+gcloud iam service-accounts add-iam-policy-binding ${ALLOYDB_USER_GSA_ID} \
+    --member "serviceAccount:${PROJECT_ID}.svc.id.goog[default/productcatalogservice]" \
+    --role roles/iam.workloadIdentityUser

kustomize/components/shopping-assistant/scripts/2_create_populate_alloydb_tables.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+# Set sensible defaults
+REGION=us-central1
+ALLOYDB_CLUSTER_NAME=onlineboutique-cluster
+ALLOYDB_CARTS_DATABASE_NAME=carts
+ALLOYDB_CARTS_TABLE_NAME=cart_items
+ALLOYDB_PRODUCTS_DATABASE_NAME=products
+ALLOYDB_PRODUCTS_TABLE_NAME=catalog_items
+
+# Fetch the primary and read IPs
+ALLOYDB_PRIMARY_IP=`gcloud alloydb instances list --region=${REGION} --cluster=${ALLOYDB_CLUSTER_NAME} --filter="INSTANCE_TYPE:PRIMARY" --format=flattened | sed -nE "s/ipAddress:\s*(.*)/\1/p"`
+
+# Create carts database and table
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -c "CREATE DATABASE ${ALLOYDB_CARTS_DATABASE_NAME}"
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_CARTS_DATABASE_NAME} -c "CREATE TABLE ${ALLOYDB_CARTS_TABLE_NAME} (userId text, productId text, quantity int, PRIMARY KEY(userId, productId))"
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_CARTS_DATABASE_NAME} -c "CREATE INDEX cartItemsByUserId ON ${ALLOYDB_CARTS_TABLE_NAME}(userId)"
+
+# Create products database, table, and extensions
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -c "CREATE DATABASE ${ALLOYDB_PRODUCTS_DATABASE_NAME}"
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_PRODUCTS_DATABASE_NAME} -c "CREATE EXTENSION IF NOT EXISTS vector"
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_PRODUCTS_DATABASE_NAME} -c "CREATE EXTENSION IF NOT EXISTS google_ml_integration CASCADE;"
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_PRODUCTS_DATABASE_NAME} -c "GRANT EXECUTE ON FUNCTION embedding TO postgres;"
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_PRODUCTS_DATABASE_NAME} -c "CREATE TABLE ${ALLOYDB_PRODUCTS_TABLE_NAME} (id TEXT PRIMARY KEY, name TEXT, description TEXT, picture TEXT, price_usd_currency_code TEXT, price_usd_units INTEGER, price_usd_nanos BIGINT, categories TEXT, product_embedding VECTOR(768), embed_model TEXT)"
+
+# Generate and insert products table entries
+python3 ./generate_sql_from_products.py > products.sql
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_PRODUCTS_DATABASE_NAME} -f products.sql
+rm products.sql
+
+# Generate vector embeddings
+psql -h ${ALLOYDB_PRIMARY_IP} -U postgres -d ${ALLOYDB_PRODUCTS_DATABASE_NAME} -c "UPDATE ${ALLOYDB_PRODUCTS_TABLE_NAME} SET product_embedding = embedding('textembedding-gecko@003', description), embed_model='textembedding-gecko@003';"

kustomize/components/shopping-assistant/scripts/generate_sql_from_products.py

@@ -0,0 +1,50 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+table_name = "catalog_items"
+fields = [
+    'id', 'name', 'description', 'picture', 
+    'price_usd_currency_code', 'price_usd_units', 'price_usd_nanos', 
+    'categories'
+]
+
+# Load the produts JSON
+with open("products.json", 'r') as f:
+    data = json.load(f)
+
+# Generate SQL INSERT statements
+for product in data['products']:
+    columns = ', '.join(fields)
+    placeholders = ', '.join(['{}'] * len(fields))
+    sql = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders});"
+
+    # Escape single quotes within product data 
+    product['name'] = product['name'].replace("'", "")
+    product['description'] = product['description'].replace("'", "")
+
+    escaped_values = (
+        f"'{product['id']}'",
+        f"'{product['name']}'",
+        f"'{product['description']}'",
+        f"'{product['picture']}'",
+        f"'{product['priceUsd']['currencyCode']}'",
+        product['priceUsd']['units'],
+        product['priceUsd']['nanos'],
+        f"'{','.join(product['categories'])}'"
+    )
+
+    # Render the formatted SQL query
+    print(sql.format(*escaped_values))