# HG changeset patch # User drewp@bigasterisk.com # Date 2023-06-08 17:50:01 # Node ID b53ab97e8979f70ee73356386e06fb0ffcfdca65 # Parent 695948b426aefc06872b27aa5cb395ba016b5738 reorganize, and add two retry loops to try to get everything to startup in one 'inv run' diff --git a/.style.yapf b/.style.yapf new file mode 100644 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,4 @@ +# overwritten by /home/drewp/bin/setup_home_venv +[style] +based_on_style = google +column_limit = 160 diff --git a/20-kube/10-pom-pom.yaml b/20-kube/10-pom-pom.yaml deleted file mode 100644 --- a/20-kube/10-pom-pom.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: ingress.pomerium.io/v1 -kind: Pomerium -metadata: - name: global -spec: - secrets: pomerium/bootstrap - authenticate: - url: https://authenticate.bigasterisk.com - cookie: - expire: 20h - identityProvider: - provider: oidc - url: https://accounts.google.com - scopes: - - openid - - email - # adds name+locale to user details - - profile - secret: pomerium/idp - storage: - postgres: - secret: pomerium/postgres-connection-key - # Note pom won't start up if this cert doesn't exist, so you have to run once - # with it commented out, then after cert success, run again with it enabled. - certificates: [pomerium/pomerium-proxy-tls] diff --git a/20-kube/06-postgres.yaml b/config/06-postgres.yaml rename from 20-kube/06-postgres.yaml rename to config/06-postgres.yaml diff --git a/30-cert-manager/51-pomerium-production-issuer.yaml b/config/51-pomerium-production-issuer.yaml rename from 30-cert-manager/51-pomerium-production-issuer.yaml rename to config/51-pomerium-production-issuer.yaml diff --git a/30-cert-manager/51-pomerium-staging-issuer.yaml b/config/51-pomerium-staging-issuer.yaml rename from 30-cert-manager/51-pomerium-staging-issuer.yaml rename to config/51-pomerium-staging-issuer.yaml diff --git a/30-cert-manager/60-auth-cert.yaml b/config/60-auth-cert.yaml rename from 30-cert-manager/60-auth-cert.yaml rename to config/60-auth-cert.yaml diff --git a/10-vols/claims.yaml b/config/claims.yaml rename from 10-vols/claims.yaml rename to config/claims.yaml diff --git a/kustomization.yaml b/config/kustomization.yaml rename from kustomization.yaml rename to config/kustomization.yaml --- a/kustomization.yaml +++ b/config/kustomization.yaml @@ -1,22 +1,6 @@ bases: - - ingress-controller/deployment.yaml - - cert-manager-v1.12.0.yaml - - 10-vols/volumes.yaml - - 10-vols/claims.yaml - - 20-kube/05-idp-secret.yaml - - 20-kube/06-postgres.yaml - - 30-cert-manager/60-auth-cert.yaml - - 30-cert-manager/51-pomerium-production-issuer.yaml - - 30-cert-manager/51-pomerium-staging-issuer.yaml - - 20-kube/10-pom-pom.yaml -patchesStrategicMerge: - - "patch.yaml" -patches: - - target: - kind: Deployment - name: pomerium - namespace: pomerium - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: "--debug" \ No newline at end of file + - volumes.yaml + - claims.yaml + - 05-idp-secret.yaml + - 06-postgres.yaml + \ No newline at end of file diff --git a/10-vols/volumes.yaml b/config/volumes.yaml rename from 10-vols/volumes.yaml rename to config/volumes.yaml diff --git a/ingress-default.yaml b/ingress/default.yaml rename from ingress-default.yaml rename to ingress/default.yaml diff --git a/ingress-static.yaml b/ingress/static.yaml rename from ingress-static.yaml rename to ingress/static.yaml diff --git a/make_global.py b/make_global.py new file mode 100755 --- /dev/null +++ b/make_global.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3 + +import json +import subprocess +import sys +import time + + +def getSuffixedName() -> str: + ns = 'pomerium' + j = json.loads(subprocess.check_output(["kubectl", "get", "-n", ns, "secret", "-o", "json"]).decode('utf8')) + for item in j['items']: + name = item['metadata']['name'] + if name.startswith('pomerium-proxy-tls-'): + return ns + '/' + name + raise ValueError() + + +config = { + 'apiVersion': "ingress.pomerium.io/v1", + 'kind': "Pomerium", + 'metadata': { + 'name': "global" + }, + 'spec': { + 'secrets': "pomerium/bootstrap", + 'authenticate': { + 'url': "https://authenticate.bigasterisk.com" + }, + 'cookie': { + 'expire': "20h" + }, + 'identityProvider': { + 'provider': "oidc", + 'url': "https://accounts.google.com", + 'scopes': [ + "openid", + "email", + "profile" # adds name+locale to user details + ], + 'secret': "pomerium/idp" + }, + 'storage': { + 'postgres': { + 'secret': "pomerium/postgres-connection-key" + } + }, + } +} + +# Old note: pom won't start up if this cert doesn't exist, so you have to run once +# with it commented out, then after cert success, run again with it enabled. + +sys.stderr.write("wait for secret: ") +for tries in range(100): + try: + config['spec']['certificates'] = [ + #getSuffixedName() + 'pomerium/pomerium-proxy-tls' + ] + except ValueError: + sys.stderr.write('.') + sys.stderr.flush() + time.sleep(10) + else: + break +else: + raise ValueError + +sys.stderr.write('\n') + +print(json.dumps(config)) \ No newline at end of file diff --git a/tasks.py b/tasks.py --- a/tasks.py +++ b/tasks.py @@ -1,19 +1,43 @@ +import sys +import time from invoke import task from invoke.exceptions import UnexpectedExit + +def authCert(ctx): + for tries in range(100): + try: + ctx.run("kubectl apply -f config/60-auth-cert.yaml", echo=True, ) + sys.stderr.write("worked") + return + except UnexpectedExit: + time.sleep(2) + sys.stderr.write('.') + sys.stderr.flush() + raise ValueError + + + @task def run(ctx): - ctx.run("cd 00-defs; skaffold run", echo=True) - ctx.run("cd 10-vols; skaffold run", echo=True) - ctx.run("cd 20-kube; skaffold run", echo=True) - # here we must wait for cert-manager-webhook.cert-manager.svc - ctx.run("cd 30-cert-manager; skaffold run", echo=True, warn=True) - ctx.run("cd 30-cert-manager; skaffold run", echo=True) + ctx.run("kubectl delete -n pomerium job/pomerium-gen-secrets --ignore-not-found", echo=True) + ctx.run("skaffold run -f use-invoke-not-skaffold.yaml", echo=True) + authCert(ctx) + ctx.run("./make_global.py | kubectl apply -f -", echo=True) + ctx.run("kubectl apply -f config/51-pomerium-production-issuer.yaml", echo=True) + ctx.run("kubectl apply -f config/51-pomerium-staging-issuer.yaml", echo=True) + - try: - ctx.run("kubectl get -n pomerium ingress | grep 80") - except UnexpectedExit: - raise SystemExit("expected cm-acme-http-solver-... ingress on port 80") +@task +def delete(ctx): + # todo don't delete certs that have big timeouts to remake + ctx.run("kubectl delete -n pomerium job/pomerium-gen-secrets --ignore-not-found", echo=True) + ctx.run("skaffold delete -f use-invoke-not-skaffold.yaml ", echo=True) + ctx.run("kubectl delete pomerium/global --ignore-not-found", echo=True) + ctx.run("kubectl delete -f config/60-auth-cert.yaml --ignore-not-found", echo=True) + ctx.run("kubectl delete -f config/51-pomerium-production-issuer.yaml --ignore-not-found", echo=True) + ctx.run("kubectl delete -f config/51-pomerium-staging-issuer.yaml --ignore-not-found", echo=True) + ''' troubleshooting, based on diff --git a/cert-manager-v1.12.0.yaml b/upstream/cert-manager-v1.12.0.yaml rename from cert-manager-v1.12.0.yaml rename to upstream/cert-manager-v1.12.0.yaml diff --git a/upstream/kustomization.yaml b/upstream/kustomization.yaml new file mode 100644 --- /dev/null +++ b/upstream/kustomization.yaml @@ -0,0 +1,15 @@ +bases: + - pomerium-ingress-controller.yaml + - cert-manager-v1.12.0.yaml +patchesStrategicMerge: + - "patch.yaml" + +patches: + - target: + kind: Deployment + name: pomerium + namespace: pomerium + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: "--debug" diff --git a/patch.yaml b/upstream/patch.yaml rename from patch.yaml rename to upstream/patch.yaml diff --git a/upstream/pomerium-ingress-controller.yaml b/upstream/pomerium-ingress-controller.yaml new file mode 100644 --- /dev/null +++ b/upstream/pomerium-ingress-controller.yaml @@ -0,0 +1,671 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: pomerium + name: pomerium.ingress.pomerium.io +spec: + group: ingress.pomerium.io + names: + kind: Pomerium + listKind: PomeriumList + plural: pomerium + singular: pomerium + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Pomerium define runtime-configurable Pomerium settings that do + not fall into the category of deployment parameters + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PomeriumSpec defines Pomerium-specific configuration parameters. + properties: + authenticate: + description: Authenticate sets authenticate service parameters. If + not specified, a Pomerium-hosted authenticate service would be used. + properties: + callbackPath: + description: "CallbackPath sets the path at which the authenticate + service receives callback responses from your identity provider. + The value must exactly match one of the authorized redirect + URIs for the OAuth 2.0 client. \n

This value is referred + to as the redirect_url in the OpenIDConnect and OAuth2 specs.

+

Defaults to /oauth2/callback

" + type: string + url: + description: "AuthenticateURL is a dedicated domain URL the non-authenticated + persons would be referred to. \n

" + format: uri + pattern: ^https:// + type: string + required: + - url + type: object + caSecrets: + description: CASecret should refer to k8s secrets with key ca.crt + containing a CA certificate. + items: + type: string + type: array + certificates: + description: Certificates is a list of secrets of type TLS to use + format: namespace/name + items: + type: string + type: array + cookie: + description: Cookie defines Pomerium session cookie options. + properties: + domain: + description: Domain defaults to the same host that set the cookie. + If you specify the domain explicitly, then subdomains would + also be included. + type: string + expire: + description: Expire sets cookie and Pomerium session expiration + time. Once session expires, users would have to re-login. If + you change this parameter, existing sessions are not affected. +

See Session + Management (Enterprise) for a more fine-grained session + controls.

Defaults to 14 hours.

+ format: duration + type: string + httpOnly: + description: HTTPOnly if set to false, the cookie + would be accessible from within the JavaScript. Defaults to + true. + type: boolean + name: + description: Name sets the Pomerium session cookie name. Defaults + to _pomerium + type: string + sameSite: + description: SameSite sets the SameSite option for cookies. Defaults + to . + type: string + secure: + description: Secure if set to false, would make a cookie accessible + over insecure protocols (HTTP). Defaults to true. + type: boolean + type: object + identityProvider: + description: IdentityProvider configure single-sign-on authentication + and user identity details by integrating with your Identity + Provider + properties: + provider: + description: Provider is the short-hand name of a built-in OpenID + Connect (oidc) identity provider to be used for authentication. + To use a generic provider, set to oidc. + enum: + - auth0 + - azure + - github + - gitlab + - google + - oidc + - okta + - onelogin + - ping + type: string + refreshDirectory: + description: RefreshDirectory is no longer supported, please see + Upgrade + Guide. + properties: + interval: + description: interval is the time that pomerium will sync + your IDP directory. + format: duration + type: string + timeout: + description: timeout is the maximum time allowed each run. + format: duration + type: string + required: + - interval + - timeout + type: object + requestParams: + additionalProperties: + type: string + description: RequestParams to be added as part of a sign-in request + using OAuth2 code flow. + format: namespace/name + type: object + requestParamsSecret: + description: RequestParamsSecret is a reference to a secret for + additional parameters you'd prefer not to provide in plaintext. + format: namespace/name + type: string + scopes: + description: Scopes Identity provider scopes correspond to access + privilege scopes as defined in Section 3.3 of OAuth 2.0 RFC6749. + items: + type: string + type: array + secret: + description: Secret containing IdP provider specific parameters. + and must contain at least client_id and client_secret + values. + format: namespace/name + minLength: 1 + type: string + serviceAccountFromSecret: + description: ServiceAccountFromSecret is no longer supported, + see Upgrade + Guide. + type: string + url: + description: URL is the base path to an identity provider's OpenID + connect discovery document. See Identity + Providers guides for details. + format: uri + pattern: ^https:// + type: string + required: + - provider + - secret + type: object + jwtClaimHeaders: + additionalProperties: + type: string + description: JWTClaimHeaders convert claims from the assertion token + into HTTP headers and adds them into JWT assertion header. Please + make sure to read + Getting User Identity guide. + type: object + programmaticRedirectDomains: + description: ProgrammaticRedirectDomains specifies a list of domains + that can be used for programmatic + redirects. + items: + type: string + type: array + secrets: + description: "Secrets references a Secret with Pomerium bootstrap + parameters. \n

In a default + Pomerium installation manifest, they would be generated via a one-time + job and stored in a pomerium/bootstrap Secret. + You may re-run the job to rotate the secrets, or update the Secret + values manually.

" + format: namespace/name + minLength: 1 + type: string + setResponseHeaders: + additionalProperties: + type: string + description: SetResponseHeaders specifies a mapping of HTTP Header + to be added globally to all managed routes and pomerium's authenticate + service. See Set + Response Headers + type: object + storage: + description: Storage defines persistent storage for sessions and other + data. See Storage + for details. If no storage is specified, Pomerium would use a transient + in-memory storage (not recommended for production). + properties: + postgres: + description: Postgres specifies PostgreSQL database connection + parameters + properties: + caSecret: + description: CASecret should refer to a k8s secret with key + ca.crt containing CA certificate that, if specified, + would be used to populate sslrootcert parameter + of the connection string. + format: namespace/name + minLength: 1 + type: string + secret: + description: Secret specifies a name of a Secret that must + contain connection key. See DSN + Format and Parameters. Do not set sslrootcert, + sslcert and sslkey via connection + string, use tlsSecret and caSecret + CRD options instead. + format: namespace/name + minLength: 1 + type: string + tlsSecret: + description: TLSSecret should refer to a k8s secret of type + kubernetes.io/tls and allows to specify an + optional client certificate and key, by constructing sslcert + and sslkey connection string + parameter values. + format: namespace/name + minLength: 1 + type: string + required: + - secret + type: object + redis: + description: Redis defines REDIS connection parameters + properties: + caSecret: + description: CASecret should refer to a k8s secret with key + ca.crt that must be a PEM-encoded certificate + authority to use when connecting to the databroker storage + engine. + format: namespace/name + type: string + secret: + description: Secret specifies a name of a Secret that must + contain connection key. + format: namespace/name + minLength: 1 + type: string + tlsSecret: + description: TLSSecret should refer to a k8s secret of type + kubernetes.io/tls that would be used to perform + TLS connection to REDIS. + format: namespace/name + minLength: 1 + type: string + tlsSkipVerify: + description: TLSSkipVerify disables TLS certificate chain + validation. + type: boolean + required: + - secret + type: object + type: object + required: + - secrets + type: object + status: + description: PomeriumStatus represents configuration and Ingress status. + properties: + ingress: + additionalProperties: + description: ResourceStatus represents the outcome of the latest + attempt to reconcile relevant Kubernetes resource with Pomerium. + properties: + error: + description: Error that prevented latest observedGeneration + to be synchronized with Pomerium. + type: string + observedAt: + description: ObservedAt is when last reconciliation attempt + was made. + format: date-time + type: string + observedGeneration: + description: ObservedGeneration represents the .metadata.generation + that was last presented to Pomerium. + format: int64 + type: integer + reconciled: + description: Reconciled is whether this object generation was + successfully synced with pomerium. + type: boolean + warnings: + description: Warnings while parsing the resource. + items: + type: string + type: array + required: + - reconciled + type: object + description: Routes provide per-Ingress status. + type: object + settingsStatus: + description: SettingsStatus represent most recent main configuration + reconciliation status. + properties: + error: + description: Error that prevented latest observedGeneration to + be synchronized with Pomerium. + type: string + observedAt: + description: ObservedAt is when last reconciliation attempt was + made. + format: date-time + type: string + observedGeneration: + description: ObservedGeneration represents the .metadata.generation + that was last presented to Pomerium. + format: int64 + type: integer + reconciled: + description: Reconciled is whether this object generation was + successfully synced with pomerium. + type: boolean + warnings: + description: Warnings while parsing the resource. + items: + type: string + type: array + required: + - reconciled + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-controller + namespace: pomerium +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-gen-secrets + namespace: pomerium +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-controller +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services/status + - secrets/status + - endpoints/status + verbs: + - get +- apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - ingress.pomerium.io + resources: + - pomerium + verbs: + - get + - list + - watch +- apiGroups: + - ingress.pomerium.io + resources: + - pomerium/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-gen-secrets +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pomerium-controller +subjects: +- kind: ServiceAccount + name: pomerium-controller + namespace: pomerium +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-gen-secrets +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pomerium-gen-secrets +subjects: +- kind: ServiceAccount + name: pomerium-gen-secrets + namespace: pomerium +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-metrics + namespace: pomerium +spec: + ports: + - name: metrics + port: 9090 + protocol: TCP + targetPort: metrics + selector: + app.kubernetes.io/name: pomerium + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-proxy + namespace: pomerium +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/name: pomerium + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium + namespace: pomerium +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pomerium + template: + metadata: + labels: + app.kubernetes.io/name: pomerium + spec: + containers: + - args: + - all-in-one + - --pomerium-config=global + - --update-status-from-service=$(POMERIUM_NAMESPACE)/pomerium-proxy + - --metrics-bind-address=$(POD_IP):9090 + env: + - name: TMPDIR + value: /tmp + - name: XDG_CACHE_HOME + value: /tmp + - name: POMERIUM_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: pomerium/ingress-controller:main + imagePullPolicy: Always + name: pomerium + ports: + - containerPort: 8443 + name: https + protocol: TCP + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP + resources: + limits: + cpu: 5000m + memory: 1Gi + requests: + cpu: 300m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + serviceAccountName: pomerium-controller + terminationGracePeriodSeconds: 10 + volumes: + - emptyDir: {} + name: tmp +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-gen-secrets + namespace: pomerium +spec: + template: + metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium-gen-secrets + spec: + containers: + - args: + - gen-secrets + - --secrets=$(POD_NAMESPACE)/bootstrap + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: pomerium/ingress-controller:main + imagePullPolicy: IfNotPresent + name: gen-secrets + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: pomerium-gen-secrets +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/name: pomerium + name: pomerium +spec: + controller: pomerium.io/ingress-controller diff --git a/skaffold.yaml b/use-invoke-not-skaffold.yaml rename from skaffold.yaml rename to use-invoke-not-skaffold.yaml --- a/skaffold.yaml +++ b/use-invoke-not-skaffold.yaml @@ -5,7 +5,8 @@ metadata: manifests: kustomize: paths: - - "." + - "upstream" + - "config" rawYaml: - - ingress-default.yaml - - ingress-static.yaml + - ingress/default.yaml + - ingress/static.yaml