--- a/create_k8s.py	Fri May 03 11:56:59 2024 -0700
+++ b/create_k8s.py	Fri May 03 12:27:16 2024 -0700
@@ -212,11 +212,288 @@
 
 
 def createAlertObjs(objPrefix, webRoot):
-    (build / f'{objPrefix}-4-vmalert_0pv.yaml').write_text(
-        toJson(
+    (build / f'{objPrefix}-4vmalert_0pv.yaml').write_text(
+        toJson({
+            "apiVersion": "v1",
+            "kind": "PersistentVolume",
+            "metadata": {
+                "name": "opt-alertmanager",
+                "labels": {
+                    "type": "local"
+                }
+            },
+            "spec": {
+                "storageClassName": "manual",
+                "hostPath": {
+                    "path": "/opt/alertmanager"
+                },
+                "capacity": {
+                    "storage": "50Gi"
+                },
+                "accessModes": ["ReadWriteOnce"],
+                "persistentVolumeReclaimPolicy": "Retain",
+                "claimRef": {
+                    "namespace": "default",
+                    "name": "opt-alertmanager"
+                }
+            }
+        }))
+    (build / f'{objPrefix}-4vmalert_1pvc.yaml').write_text(toJson({
+        "apiVersion": "v1",
+        "kind": "PersistentVolumeClaim",
+        "metadata": {
+            "name": "opt-alertmanager"
+        },
+        "spec": {
+            "storageClassName": "",
+            "volumeName": "opt-alertmanager",
+            "accessModes": ["ReadWriteOnce"],
+            "resources": {
+                "requests": {
+                    "storage": "50Gi"
+                }
+            }
+        },
+    }))
+    (build / f'{objPrefix}-4vmalert_2deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1",
+            "kind": "Deployment",
+            "metadata": {
+                "name": "vmalert"
+            },
+            "spec": {
+                "replicas": 1,
+                "strategy": {
+                    "type": "Recreate"
+                },
+                "selector": {
+                    "matchLabels": {
+                        "app": "vmalert"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app": "vmalert"
+                        },
+                        "annotations": {
+                            "prometheus.io/scrape": "true"
+                        }
+                    },
+                    "spec": {
+                        "volumes": [{
+                            "name": "config",
+                            "configMap": {
+                                "name": "victoriametrics-config"
+                            }
+                        }],
+                        "serviceAccountName":
+                            "victoriametrics",
+                        "containers": [{
+                            "name": "vmalert",
+                            "image": "docker.io/victoriametrics/vmalert:v1.91.2",
+                            "args": [
+                                "-configCheckInterval=5s",
+                                "-datasource.url=http://victoriametrics/m/",
+                                "-datasource.queryStep=5m",
+                                "-evaluationInterval=1m",
+                                "-external.url=https://bigasterisk.com/vmalert",
+                                "-loggerLevel=INFO",
+                                "-loggerTimezone=America/Los_Angeles",
+                                "-memory.allowedBytes=512MB",
+                                "-notifier.url=http://alertmanager",
+                                "-remoteRead.url=http://victoriametrics/m/",
+                                "-remoteWrite.url=http://victoriametrics/m/",
+                                "-rule=/local/rules",
+                            ],
+                            "ports": [{
+                                "containerPort": 8880
+                            }],
+                            "volumeMounts": [{
+                                "name": "config",
+                                "mountPath": "/local"
+                            }]
+                        }]
+                    }
+                }
+            }
+        }))
 
-        )
-    )
+    (build / f'{objPrefix}-4vmalert_svc.yaml').write_text(toJson({
+        "apiVersion": "v1",
+        "kind": "Service",
+        "metadata": {
+            "name": "vmalert"
+        },
+        "spec": {
+            "ports": [{
+                "port": 80,
+                "targetPort": 8880
+            }],
+            "selector": {
+                "app": "vmalert"
+            }
+        },
+    }))
+    (build / f'{objPrefix}-5alertmanager_deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1",
+            "kind": "Deployment",
+            "metadata": {
+                "name": "alertmanager"
+            },
+            "spec": {
+                "replicas": 1,
+                "selector": {
+                    "matchLabels": {
+                        "app": "alertmanager"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app": "alertmanager"
+                        }
+                    },
+                    "spec": {
+                        "volumes": [{
+                            "name": "opt-alertmanager",
+                            "persistentVolumeClaim": {
+                                "claimName": "opt-alertmanager"
+                            }
+                        }],
+                        "serviceAccountName": "victoriametrics",
+                        "containers": [{
+                            "name": "alertmanager",
+                            "image": "docker.io/prom/alertmanager:v0.27.0",
+                            "args": [
+                                "--config.file=/alertmanager/alertmanager.yml",
+                                "--web.external-url=https://bigasterisk.com/alertmanager/",
+                                "--web.route-prefix=/",
+                                "--log.level=info",
+                            ],
+                            "ports": [{
+                                "containerPort": 9093
+                            }],
+                            "volumeMounts": [{
+                                "name": "opt-alertmanager",
+                                "mountPath": "/alertmanager"
+                            }]
+                        }],
+                        "affinity": {
+                            "nodeAffinity": {
+                                "requiredDuringSchedulingIgnoredDuringExecution": {
+                                    "nodeSelectorTerms": [{
+                                        "matchExpressions": [{
+                                            "key": "kubernetes.io/hostname",
+                                            "operator": "In",
+                                            "values": ["ditto"]
+                                        }]
+                                    }]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }))
+    (build / f'{objPrefix}-5alertmanager_svc.yaml').write_text(toJson({
+        "apiVersion": "v1",
+        "kind": "Service",
+        "metadata": {
+            "name": "alertmanager"
+        },
+        "spec": {
+            "ports": [{
+                "port": 80,
+                "targetPort": 9093
+            }],
+            "selector": {
+                "app": "alertmanager"
+            }
+        },
+    }))
+    (build / f'{objPrefix}-4vmalert_ingress.yaml').write_text(
+        toJson({
+            "apiVersion": "networking.k8s.io/v1",
+            "kind": "Ingress",
+            "metadata": {
+                "name": "vmalert",
+                "annotations": {
+                    "cert-manager.io/cluster-issuer": "letsencrypt-prod",
+                    "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
+                    "ingress.pomerium.io/pass_identity_headers": "true",
+                    "ingress.pomerium.io/preserve_host_header": "true",
+                    "ingress.pomerium.io/policy": "allow:\n  or: \n    - { email: { is: \"drewpca@gmail.com\" }}\n    - { email: { is: \"kelsimp@gmail.com\" }}\n"
+                }
+            },
+            "spec": {
+                "ingressClassName": "pomerium",
+                "rules": [{
+                    "host": "bigasterisk.com",
+                    "http": {
+                        "paths": [{
+                            "pathType": "Prefix",
+                            "path": "/vmalert/",
+                            "backend": {
+                                "service": {
+                                    "name": "vmalert",
+                                    "port": {
+                                        "number": 80
+                                    }
+                                }
+                            }
+                        }]
+                    }
+                }],
+                "tls": [{
+                    "hosts": ["bigasterisk.com"],
+                    "secretName": "bigasterisk.com-tls"
+                }]
+            }
+        }))
+    (build / f'{objPrefix}-5alertmanager_ingress.yaml').write_text(
+        toJson({
+            "apiVersion": "networking.k8s.io/v1",
+            "kind": "Ingress",
+            "metadata": {
+                "name": "alertmanager",
+                "annotations": {
+                    "cert-manager.io/cluster-issuer": "letsencrypt-prod",
+                    "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
+                    "ingress.pomerium.io/pass_identity_headers": "true",
+                    "ingress.pomerium.io/preserve_host_header": "true",
+                    "ingress.pomerium.io/policy": "allow:\n  or: \n    - { email: { is: \"drewpca@gmail.com\" }}\n    - { email: { is: \"kelsimp@gmail.com\" }}\n",
+                    "ingress.pomerium.io/prefix_rewrite": "/"
+                }
+            },
+            "spec": {
+                "ingressClassName": "pomerium",
+                "rules": [{
+                    "host": "bigasterisk.com",
+                    "http": {
+                        "paths": [{
+                            "pathType": "Prefix",
+                            "path": "/alertmanager/",
+                            "backend": {
+                                "service": {
+                                    "name": "alertmanager",
+                                    "port": {
+                                        "number": 80
+                                    }
+                                }
+                            }
+                        }]
+                    }
+                }],
+                "tls": [{
+                    "hosts": ["bigasterisk.com"],
+                    "secretName": "bigasterisk.com-tls"
+                }]
+            }
+        }))
+
 
 def main():
     tzArg = "-loggerTimezone=America/Los_Angeles"
@@ -255,6 +532,7 @@
     createIndex(objPrefix, webRoot, makeIndexHtml(objPrefix, webRoot, webHost))
     createAlertObjs(objPrefix, webRoot)
 
+
 main()
 
 # in vmui, set server url to