Changeset - 54b0edb7cca8
Parent rev.
Child rev.
[Not reviewed]
default
0 1 1
drewp@bigasterisk.com - 22 months ago 2022-12-13 07:24:30
drewp@bigasterisk.com
debug notes
2 files changed with 76 insertions and 0 deletions:
readme
48
tasks.py
28
0 comments (0 inline, 0 general)
readme
➞
Show inline comments
 
new file 100644
 

			




	
	
	
		
 
https://www.pomerium.com/docs/k8s/quickstart

			




	
	
	
		
 

			




	
	
	
		
 
kubectl apply -f deployment.yaml

			




	
	
	
		
 

			




	
	
	
		
 
3152  sudo apt install libnss3-tools

			




	
	
	
		
 
 3153  ./mkcert-v1.4.4-linux-amd64 -install

			




	
	
	
		
 
 3156  ./mkcert-v1.4.4-linux-amd64 "*.localhost.pomerium.io"

			




	
	
	
		
 
 3158  kubectl create secret tls pomerium-wildcard-tls --namespace=pomerium  --cert=./_wildcard.localhost.pomerium.io.pem --key=./_wildcard.localhost.pomerium.io-key.pem

			




	
	
	
		
 

			




	
	
	
		
 
 k rollout restart -n pomerium deploy/pomerium

			




	
	
	
		
 

			




	
	
	
		
 
----------------------------------------------------------------

			




	
	
	
		
 
bootstrap:

			




	
	
	
		
 
comment out 10-pomerium.yaml certificates line.

			




	
	
	
		
 

			




	
	
	
		
 
get to this saying ready=true

			




	
	
	
		
 
k get -n pomerium certificate/pomerium-proxy-tls -o wide

			




	
	
	
		
 

			




	
	
	
		
 
enable 10-pomerium.yaml certificates line.

			




	
	
	
		
 
k apply -f kube/10-pomerium.yaml

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
✨ dash(pts/11):/my/serv/pomerium% k apply -f verify-service.yaml

			




	
	
	
		
 
service/verify created

			




	
	
	
		
 
deployment.apps/verify created

			




	
	
	
		
 
✨ dash(pts/11):/my/serv/pomerium% k apply -f verify-ingress.yaml 

			




	
	
	
		
 

			




	
	
	
		
 
k get -A certificate -o wide

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
todo:

			




	
	
	
		
 
https://www.pomerium.com/docs/topics/data-storage#postgres

			




	
	
	
		
 

			




	
	
	
		
 
---------------------------------------------

			




	
	
	
		
 
2022-12-11

			




	
	
	
		
 

			




	
	
	
		
 
inv run

			




	
	
	
		
 

			




	
	
	
		
 
-------------

			




	
	
	
		
 

			




	
	
	
		
 
I1212 18:37:55.559944       1 pod.go:59] cert-manager/challenges/http01/selfCheck/http01/ensurePod "msg"="found one existing HTTP01 solver pod" "dnsName"="authenticate.bigasterisk.com" "related_resource_kind"="Pod" "related_resource_name"="cm-acme-http-solver-szbwz" "related_resource_namespace"="pomerium" "related_resource_version"="v1" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"

			




	
	
	
		
 

			




	
	
	
		
 
I1212 18:37:55.561255       1 service.go:43] cert-manager/challenges/http01/selfCheck/http01/ensureService "msg"="found one existing HTTP01 solver Service for challenge resource" "dnsName"="authenticate.bigasterisk.com" "related_resource_kind"="Service" "related_resource_name"="cm-acme-http-solver-gw5dd" "related_resource_namespace"="pomerium" "related_resource_version"="v1" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"

			




	
	
	
		
 

			




	
	
	
		
 
I1212 18:37:55.562467       1 ingress.go:99] cert-manager/challenges/http01/selfCheck/http01/ensureIngress "msg"="found one existing HTTP01 solver ingress" "dnsName"="authenticate.bigasterisk.com" "related_resource_kind"="Ingress" "related_resource_name"="cm-acme-http-solver-skn9b" "related_resource_namespace"="pomerium" "related_resource_version"="v1" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"

			




	
	
	
		
 

			




	
	
	
		
 
E1212 18:37:55.604107       1 sync.go:190] cert-manager/challenges "msg"="propagation check failed" "error"="failed to perform self check GET request 'http://authenticate.bigasterisk.com/.well-known/acme-challenge/40xNPGvj7rFgf1PSlm4UKmQw6i9BJsMe8orpIFXrxAc': Get \"http://authenticate.bigasterisk.com/.well-known/acme-challenge/40xNPGvj7rFgf1PSlm4UKmQw6i9BJsMe8orpIFXrxAc\": EOF" "dnsName"="authenticate.bigasterisk.com" "resource_kind"="Challenge" "resource_name"="pomerium-proxy-tls-5hmnq-2918954797-1564766882" "resource_namespace"="pomerium" "resource_version"="v1" "type"="HTTP-01"

			




        

    


    
    

    

        

                

                    tasks.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
	
		
 
from invoke import task

			




	
	
	
		
 
from invoke.exceptions import UnexpectedExit

			




	
	
	
		
 
@task

			




	
	
	
		
 
def delete(ctx):

			




	
	
	
		
 
    ctx.run("cd 30-cert-manager; skaffold delete")

			




	
	
	
		
 
    ctx.run("cd 20-kube; skaffold delete")

			




	
	
	
		
 
    ctx.run("cd 10-vols; skaffold delete")

			




	
	
	
		
 
    ctx.run("cd 00-defs; skaffold delete")

			




	
	
	
		
 

			




	
	
	
		
 
@task

			




	
	
	
		
 
def run(ctx):

			




	
	
	
		
 
    ctx.run("cd 00-defs; skaffold run", echo=True)

			




	
	
	
		
 
    ctx.run("cd 10-vols; skaffold run", echo=True)

			




	
	
	
		
 
    ctx.run("cd 20-kube; skaffold run", echo=True)

			




	
	
	
		
 
    # here we must wait for cert-manager-webhook.cert-manager.svc

			




	
	
	
		
 
    ctx.run("cd 30-cert-manager; skaffold run", echo=True, warn=True)

			




	
	
	
		
 
    ctx.run("cd 30-cert-manager; skaffold run", echo=True)

			




	
	
	
		
 

			




	
	
	
		
 
    try:

			




	
	
	
		
 
        ctx.run("kubectl get -n pomerium ingress | grep 80")

			




	
	
	
		
 
    except UnexpectedExit:

			




	
	
	
		
 
        raise SystemExit("expected cm-acme-http-solver-... ingress on port 80")

			




	
	
	
		
 

			




	
	
	
		
 
'''

			




	
	
	
		
 
troubleshooting, based on 

			




	
	
	
		
 
https://cert-manager.io/docs/troubleshooting/

			




	
	
	
		
 
then

			




	
	
	
		
 
https://cert-manager.io/docs/concepts/acme-orders-challenges/

			




	
	
	
		
 

			




	
	
	
		
 
I had these open:

			




	
	
	
		
 
✨ dash(pts/31):~% watch 'kubectl describe -n pomerium issuers.cert-manager.io letsencrypt-staging'

			




	
	
	
		
 
✨ dash(pts/31):~% watch 'kubectl describe -n pomerium issuers.cert-manager.io letsencrypt-prod'

			




	
	
	
		
 
✨ dash(pts/29):~% watch "kubectl get -n pomerium certificates.cert-manager.io -o wide"

			




	
	
	
		
 
✨ dash(pts/36):~% watch 'kubectl describe -n pomerium certificaterequests.cert-manager.io'

			




	
	
	
		
 
✨ dash(pts/37):~% watch 'kubectl describe -n pomerium orders.acme.cert-manager.io'

			




	
	
	
		
 
✨ dash(pts/38):~% watch 'kubectl describe -n pomerium challenges.acme.cert-manager.io '

			




	
	
	
		
 

			




	
	
	
		
 
then i checked clusterissuer vs issuer, the ns of the 60-auth-cert.yaml resources,

			




	
	
	
		
 
and i often restarted cert-manager and eventually pomerium too. 10-pom-pom.yaml last line 

			




	
	
	
		
 
may need to be toggled.

			




	
	
	
		
 

			




	
	
	
		
 
The 'cm-acme-http-solver' ingress for LE comes and goes but i didn't have to force it to exist.

			




	
	
	
		
 

			




	
	
	
		
 
Didn't need 04-gen-secrets-job.yaml

			




	
	
	
		
 

			




	
	
	
		
 
Also, CM says this a lot which means it may be afraid to renew bigasterisk.com

			




	
	
	
		
 

			




	
	
	
		
 
    I1213 07:00:01.946799       1 sync.go:394] cert-manager/controller/ingress-shim "msg"="certificate resource is not owned by this object. refusing to update non-owned certificate resource for object" "related_resource_kind"="Certificate" "related_resource_name"="bigasterisk.com-tls" "related_resource_namespace"="default" "related_resource_version"="v1" "resource_kind"="Ingress" "resource_name"="registry" "resource_namespace"="default" "resource_version"="v1"

			




	
	
	
		
 

			




	
	
	
		
 
'''

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        Server instance: kallithea-749ff669b6-v9g5f-76
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2021 by various authors & licensed under GPLv3.
            – Support