pomerium Changeset - b1f75b0584f3

Changeset - b1f75b0584f3

Parent rev.

Child rev.

[Not reviewed]

default

2 2 0

drewp@bigasterisk.com - 16 months ago 2023-06-22 06:01:32
drewp@bigasterisk.com

redo 'run' task and 'delete' (less tested)

4 files changed with 95 insertions and 84 deletions:

config/kustomization.yaml

make_global.py

tasks.py

use-invoke-not-skaffold.yaml

0 comments (0 inline, 0 general)

config/kustomization.yaml

➞

Show inline comments

deleted file

make_global.py

➞

Show inline comments

 #!/usr/bin/python3
 import json
 import subprocess
 import sys
 import time
 POM_CERT_NAME = 'pomerium-proxy-tls'
 AUTH_HOST = 'authenticate2.bigasterisk.com'
 def getSuffixedName() -> str:
     ns = 'pomerium'
 (phase,) = sys.argv[1:]
 def secretExists(qname):
     ns, localName = qname.split('/')
     j = json.loads(subprocess.check_output(["kubectl", "get", "-n", ns, "secret", "-o", "json"]).decode('utf8'))
     for item in j['items']:
         name = item['metadata']['name']
         if name.startswith('pomerium-proxy-tls'):
             return ns + '/' + name
         if name == localName:
             return
     raise ValueError()
 def retryGetSuffixedName() -> str:
     sys.stderr.write("\nwait for secret: ")
 def waitForSecret(qname):
     sys.stderr.write(f"\nwait for secret {qname}: ")
     for tries in range(100):
         try:
-            return getSuffixedName()
+            return secretExists(qname)
         except ValueError:
             sys.stderr.write('.')
             sys.stderr.flush()
             time.sleep(10)
     else:
         raise ValueError
 def pomeriumGlobalConfig():
 config = {
     'apiVersion': "ingress.pomerium.io/v1",
     'kind': "Pomerium",
     'metadata': {
         'name': "global"
     },
     'spec': {
         'secrets': "pomerium/bootstrap",
         'authenticate': {
             'url': "https://authenticate.bigasterisk.com"
         },
         'cookie': {
             'expire': "20h"
     config = {
         'apiVersion': "ingress.pomerium.io/v1",
         'kind': "Pomerium",
         'metadata': {
             'name': "global"
         },
         'identityProvider': {
             'provider': "oidc",
             'url': "https://accounts.google.com",
             'scopes': [
                 "openid",
                 "email",
                 "profile"  # adds name+locale to user details
             ],
             'secret': "pomerium/idp"
         },
 #        'storage': {
 #            'postgres': {
 #                'secret': "pomerium/postgres-connection-key"
 #            }
 #        },
         'spec': {
             'secrets': "pomerium/bootstrap",
             'authenticate': {
                 'url': f"https://{AUTH_HOST}"
             },
             'cookie': {
                 'expire': "20h"
             },
             'identityProvider': {
                 'provider': "oidc",
                 'url': "https://accounts.google.com",
                 'scopes': [
                     "openid",
                     "email",
                     "profile"  # adds name+locale to user details
                 ],
                 'secret': "pomerium/idp"
             },
             #        'storage': {
             #            'postgres': {
             #                'secret': "pomerium/postgres-connection-key"
             #            }
             #        },
+        }
+    }
     if phase == 'wait_for_cert':
         waitForSecret('pomerium/pomerium-proxy-tls')
         config['spec']['certificates'] = [f'pomerium/{POM_CERT_NAME}']
     sys.stderr.write('\n')
     return config
 def pomCert():
     return {
   "apiVersion": "cert-manager.io/v1",
   "kind": "Certificate",
   "metadata": {
     "name": POM_CERT_NAME,
     "namespace": "pomerium"
   },
   "spec": {
     "dnsNames": [
       AUTH_HOST
     ],
     "issuerRef": {
       "kind": "ClusterIssuer",
       "name": "letsencrypt-dns-prod"
     },
     "secretName": "pomerium-proxy-tls"
+  }
+}
 # Old note: pom won't start up if this cert doesn't exist, so you have to run once
 # with it commented out, then after cert success, run again with it enabled.
 if phase == 'output_pom_cert':
     output = pomCert()
 else:
     output = pomeriumGlobalConfig()
 config['spec']['certificates'] = [
     # retryGetSuffixedName() # it appear this is a temporary cert and we should set the line below then wait a few minutes
     'pomerium/pomerium-proxy-tls'
+]
 sys.stderr.write('\n')
 print(json.dumps(config))
 print(json.dumps(output))

tasks.py

➞

Show inline comments

 import sys
 import time
 from invoke import task
 from invoke.exceptions import UnexpectedExit
 def authCert(ctx):
     for tries in range(100):
         try:
             ctx.run("kubectl apply -f config/60-auth-cert.yaml", echo=True, )
             sys.stderr.write("worked")
             return
         except UnexpectedExit:
             time.sleep(2)
             sys.stderr.write('.')
             sys.stderr.flush()
     raise ValueError
 @task
 def run(ctx):
     ctx.run("kubectl delete -n pomerium job/pomerium-gen-secrets --ignore-not-found", echo=True)
     ctx.run("skaffold run -f use-invoke-not-skaffold.yaml", echo=True)
     authCert(ctx)
     ctx.run("./make_global.py | kubectl apply -f -", echo=True)
     ctx.run("kubectl kustomize upstream | kubectl apply -f -", echo=True)
     print("let CM start up")
     time.sleep(15)
     ctx.run("kubectl apply -f config/05-idp-secret.yaml", echo=True)
     ctx.run("kubectl apply -f config/dns-secret.yaml", echo=True)
     # ctx.run("kubectl apply -f config/06-postgres.yaml", echo=True)
     ctx.run("kubectl apply -f config/51-pomerium-production-issuer.yaml", echo=True)
     ctx.run("kubectl apply -f config/51-pomerium-staging-issuer.yaml", echo=True)
     ctx.run("kubectl apply -f config/dns-issuers.yaml", echo=True)
     ctx.run("./make_global.py no_cert | kubectl apply -f -", echo=True)
     ctx.run("./make_global.py output_pom_cert | kubectl apply -f -", echo=True)
     # that will make infinite certs :( Clean up the redundant requests before LE ratelimits!
     #   k delete -n pomerium certificaterequests.cert-manager.io <tab>
     ctx.run("kubectl apply -f ingress/default.yaml", echo=True)
     ctx.run("kubectl apply -f ingress/static.yaml", echo=True)
     # this may wait for
     # 1) nothing; cert+secret exist
     # 2) a letsencrypt session
     # 3) a cert-manager delay before a LE session (e.g. 45 minutes)
     ctx.run("./make_global.py wait_for_cert | kubectl apply -f -", echo=True)
 @task
 def delete(ctx):
     # todo don't delete certs that have big timeouts to remake
     ctx.run("kubectl delete pomerium/global --ignore-not-found", echo=True)
     ctx.run("kubectl delete -f config/dns-issuers.yaml --ignore-not-found", echo=True)
     ctx.run("kubectl delete -f config/51-pomerium-staging-issuer.yaml --ignore-not-found", echo=True)
     ctx.run("kubectl delete -f config/51-pomerium-production-issuer.yaml --ignore-not-found", echo=True)
     ctx.run("kubectl delete -f config/60-auth-cert.yaml --ignore-not-found", echo=True)
     ctx.run("kubectl delete pomerium/global --ignore-not-found", echo=True)
     ctx.run("skaffold delete -f use-invoke-not-skaffold.yaml ", echo=True)
     ctx.run("kubectl delete -f config/06-postgres.yaml --ignore-not-found", echo=True)
     ctx.run("kubectl delete -f config/05-idp-secret.yaml --ignore-not-found", echo=True)
     # the kustomize workloads and svcs
     for type, ns, name in [
         ('job', 'pomerium', 'pomerium-gen-secrets'),
         ('deploy', 'cert-manager', 'cert-manager'),
         ('deploy', 'cert-manager', 'cert-manager-cainjector'),
         ('deploy', 'cert-manager', 'cert-manager-webhook'),
         ('deploy', 'pomerium', 'pomerium'),
         ('service', 'cert-manager', 'cert-manager'),
         ('service', 'cert-manager', 'cert-manager-webhook'),
         ('service', 'pomerium', 'pomerium-metrics'),
         ('service', 'pomerium', 'pomerium-proxy'),
     ]:
         ctx.run(f"kubectl delete -n {ns} {type} {name} --ignore-not-found", echo=True)
     ctx.run("kubectl delete -n pomerium job/pomerium-gen-secrets --ignore-not-found", echo=True)
 '''
 troubleshooting, based on
 https://cert-manager.io/docs/troubleshooting/
 then
 https://cert-manager.io/docs/concepts/acme-orders-challenges/
 I had these open:
 ✨ dash(pts/31):~% watch 'kubectl describe -n pomerium issuers.cert-manager.io letsencrypt-staging'
 ✨ dash(pts/31):~% watch 'kubectl describe -n pomerium issuers.cert-manager.io letsencrypt-prod'
 ✨ dash(pts/29):~% watch "kubectl get -n pomerium certificates.cert-manager.io -o wide"
 ✨ dash(pts/36):~% watch 'kubectl describe -n pomerium certificaterequests.cert-manager.io'
 ✨ dash(pts/37):~% watch 'kubectl describe -n pomerium orders.acme.cert-manager.io'
 ✨ dash(pts/38):~% watch 'kubectl describe -n pomerium challenges.acme.cert-manager.io '
 then i checked clusterissuer vs issuer, the ns of the 60-auth-cert.yaml resources,
 and i often restarted cert-manager and eventually pomerium too. 10-pom-pom.yaml last line
 may need to be toggled.
 The 'cm-acme-http-solver' ingress for LE comes and goes but i didn't have to force it to exist.
 Didn't need 04-gen-secrets-job.yaml
 Also, CM says this a lot which means it may be afraid to renew bigasterisk.com
     I1213 07:00:01.946799       1 sync.go:394] cert-manager/controller/ingress-shim "msg"="certificate resource is not owned by this object. refusing to update non-owned certificate resource for object" "related_resource_kind"="Certificate" "related_resource_name"="bigasterisk.com-tls" "related_resource_namespace"="default" "related_resource_version"="v1" "resource_kind"="Ingress" "resource_name"="registry" "resource_namespace"="default" "resource_version"="v1"
 '''

use-invoke-not-skaffold.yaml

➞

Show inline comments

deleted file

0 comments (0 inline, 0 general)