--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/readme	Mon Dec 12 23:24:30 2022 -0800
@@ -0,0 +1,48 @@
+
+https://www.pomerium.com/docs/k8s/quickstart
+
+kubectl apply -f deployment.yaml
+
+3152  sudo apt install libnss3-tools
+ 3153  ./mkcert-v1.4.4-linux-amd64 -install
+ 3156  ./mkcert-v1.4.4-linux-amd64 "*.localhost.pomerium.io"
+ 3158  kubectl create secret tls pomerium-wildcard-tls --namespace=pomerium  --cert=./_wildcard.localhost.pomerium.io.pem --key=./_wildcard.localhost.pomerium.io-key.pem
+
+ k rollout restart -n pomerium deploy/pomerium
+
+----------------------------------------------------------------
+bootstrap:
+comment out 10-pomerium.yaml certificates line.
+
+get to this saying ready=true
+k get -n pomerium certificate/pomerium-proxy-tls -o wide
+
+enable 10-pomerium.yaml certificates line.
+k apply -f kube/10-pomerium.yaml
+
+
+✨ dash(pts/11):/my/serv/pomerium% k apply -f verify-service.yaml
+service/verify created
+deployment.apps/verify created
+✨ dash(pts/11):/my/serv/pomerium% k apply -f verify-ingress.yaml 
+
+k get -A certificate -o wide
+
+
+todo:
+https://www.pomerium.com/docs/topics/data-storage#postgres
+
+---------------------------------------------
+2022-12-11
+
+inv run
+
+-------------
+
+I1212 18:37:55.559944       1 pod.go:59] cert-manager/challenges/http01/selfCheck/http01/ensurePod "msg"="found one existing HTTP01 solver pod" "dnsName"="authenticate.bigasterisk.com" "related_resource_kind"="Pod" "related_resource_name"="cm-acme-http-solver-szbwz" "related_resource_namespace"="pomerium" "related_resource_version"="v1" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"
+
+I1212 18:37:55.561255       1 service.go:43] cert-manager/challenges/http01/selfCheck/http01/ensureService "msg"="found one existing HTTP01 solver Service for challenge resource" "dnsName"="authenticate.bigasterisk.com" "related_resource_kind"="Service" "related_resource_name"="cm-acme-http-solver-gw5dd" "related_resource_namespace"="pomerium" "related_resource_version"="v1" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"
+
+I1212 18:37:55.562467       1 ingress.go:99] cert-manager/challenges/http01/selfCheck/http01/ensureIngress "msg"="found one existing HTTP01 solver ingress" "dnsName"="authenticate.bigasterisk.com" "related_resource_kind"="Ingress" "related_resource_name"="cm-acme-http-solver-skn9b" "related_resource_namespace"="pomerium" "related_resource_version"="v1" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"
+
+E1212 18:37:55.604107       1 sync.go:190] cert-manager/challenges "msg"="propagation check failed" "error"="failed to perform self check GET request 'http://authenticate.bigasterisk.com/.well-known/acme-challenge/40xNPGvj7rFgf1PSlm4UKmQw6i9BJsMe8orpIFXrxAc': Get \"http://authenticate.bigasterisk.com/.well-known/acme-challenge/40xNPGvj7rFgf1PSlm4UKmQw6i9BJsMe8orpIFXrxAc\": EOF" "dnsName"="authenticate.bigasterisk.com" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"

--- a/tasks.py	Mon Dec 12 23:22:03 2022 -0800
+++ b/tasks.py	Mon Dec 12 23:24:30 2022 -0800
@@ -20,3 +20,31 @@
         ctx.run("kubectl get -n pomerium ingress | grep 80")
     except UnexpectedExit:
         raise SystemExit("expected cm-acme-http-solver-... ingress on port 80")
+
+'''
+troubleshooting, based on 
+https://cert-manager.io/docs/troubleshooting/
+then
+https://cert-manager.io/docs/concepts/acme-orders-challenges/
+
+I had these open:
+✨ dash(pts/31):~% watch 'kubectl describe -n pomerium issuers.cert-manager.io letsencrypt-staging'
+✨ dash(pts/31):~% watch 'kubectl describe -n pomerium issuers.cert-manager.io letsencrypt-prod'
+✨ dash(pts/29):~% watch "kubectl get -n pomerium certificates.cert-manager.io -o wide"
+✨ dash(pts/36):~% watch 'kubectl describe -n pomerium certificaterequests.cert-manager.io'
+✨ dash(pts/37):~% watch 'kubectl describe -n pomerium orders.acme.cert-manager.io'
+✨ dash(pts/38):~% watch 'kubectl describe -n pomerium challenges.acme.cert-manager.io '
+
+then i checked clusterissuer vs issuer, the ns of the 60-auth-cert.yaml resources,
+and i often restarted cert-manager and eventually pomerium too. 10-pom-pom.yaml last line 
+may need to be toggled.
+
+The 'cm-acme-http-solver' ingress for LE comes and goes but i didn't have to force it to exist.
+
+Didn't need 04-gen-secrets-job.yaml
+
+Also, CM says this a lot which means it may be afraid to renew bigasterisk.com
+
+    I1213 07:00:01.946799       1 sync.go:394] cert-manager/controller/ingress-shim "msg"="certificate resource is not owned by this object. refusing to update non-owned certificate resource for object" "related_resource_kind"="Certificate" "related_resource_name"="bigasterisk.com-tls" "related_resource_namespace"="default" "related_resource_version"="v1" "resource_kind"="Ingress" "resource_name"="registry" "resource_namespace"="default" "resource_version"="v1"
+
+'''