{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}

{{- $architecture := coalesce .Values.mode .Values.architecture }}
{{- if eq $architecture "distributed" }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: {{ printf "%s-worker" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }}
  namespace: {{ include "common.names.namespace" . | quote }}
  labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
    app.kubernetes.io/component: worker
spec:
  replicas: 1
  podManagementPolicy: {{ .Values.podManagementPolicy | quote }}
  serviceName: {{ printf "%s-headless" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }}
  {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }}
  selector:
    matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
      app.kubernetes.io/component: worker
  {{- if .Values.updateStrategy }}
  updateStrategy: {{- toYaml .Values.updateStrategy | nindent 4 }}
  {{- end }}
  template:
    metadata:
      {{- if .Values.podAnnotations }}
      annotations: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }}
      {{- end }}
      labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }}
        app.kubernetes.io/component: worker
    spec:
      {{- include "pytorch.imagePullSecrets" . | nindent 6 }}
      serviceAccountName: {{ include "pytorch.serviceAccountName" . }}
      automountServiceAccountToken: {{ .Values.automountServiceAccountToken }}
      {{- if .Values.hostAliases }}
      hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }}
      {{- end }}
      {{- if .Values.affinity }}
      affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.affinity "context" $) | nindent 8 }}
      {{- else }}
      affinity:
        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "worker"  "customLabels" $podLabels "context" $) | nindent 10 }}
        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "worker"  "customLabels" $podLabels "context" $) | nindent 10 }}
        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }}
      {{- end }}
      {{- if .Values.nodeSelector }}
      nodeSelector: {{- include "common.tplvalues.render" ( dict "value" .Values.nodeSelector "context" $) | nindent 8 }}
      {{- end }}
      {{- if .Values.tolerations }}
      tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }}
      {{- end }}
      {{- if .Values.priorityClassName }}
      priorityClassName: {{ .Values.priorityClassName | quote }}
      {{- end }}
      {{- if .Values.schedulerName }}
      schedulerName: {{ .Values.schedulerName | quote }}
      {{- end }}
      {{- if .Values.topologySpreadConstraints }}
      topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }}
      {{- end }}
      {{- include "pytorch.securityContext" . }}
      {{- if .Values.terminationGracePeriodSeconds }}
      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
      {{- end }}
      {{- if or .Values.cloneFilesFromGit.enabled (and .Values.volumePermissions.enabled .Values.persistence.enabled) }}
      initContainers:
        {{- if .Values.cloneFilesFromGit.enabled }}
        - name: git-clone-repository
          image: {{ include "git.image" . }}
          imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
          command:
            - /bin/bash
            - -c
            - |
              [[ -f "/opt/bitnami/scripts/git/entrypoint.sh" ]] && source "/opt/bitnami/scripts/git/entrypoint.sh"
              git clone {{ .Values.cloneFilesFromGit.repository }} --branch {{ .Values.cloneFilesFromGit.revision }} /app
          volumeMounts:
            - name: empty-dir
              mountPath: /app
              subPath: app-dir
            - name: empty-dir
              mountPath: /tmp
              subPath: tmp-dir
          {{- if .Values.cloneFilesFromGit.extraVolumeMounts }}
            {{- include "common.tplvalues.render" (dict "value" .Values.cloneFilesFromGit.extraVolumeMounts "context" $) | nindent 12 }}
          {{- end }}
        {{- end }}
        {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }}
        - name: volume-permissions
          image: "{{ template "mxnet.volumePermissions.image" . }}"
          imagePullPolicy: {{ default "" .Values.volumePermissions.image.pullPolicy | quote }}
          command:
            - /bin/bash
            - -ec
            - |
              chown -R {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.persistence.mountPath }}
          securityContext:
            runAsUser: 0
          {{- if .Values.volumePermissions.resources }}
          resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }}
          {{- else if ne .Values.volumePermissions.resourcesPreset "none" }}
          resources: {{- include "common.resources.preset" (dict "type" .Values.volumePermissions.resourcesPreset) | nindent 12 }}
          {{- end }}
          volumeMounts:
            - name: data
              mountPath: {{ .Values.persistence.mountPath }}
            - name: empty-dir
              mountPath: /tmp
              subPath: tmp-dir
        {{- end }}
      {{- end }}
      containers:
        - name: worker
          image: {{ include "pytorch.image" . }}
          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
          {{- if .Values.containerSecurityContext.enabled }}
          securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.containerSecurityContext "context" $) | nindent 12 }}
          {{- end }}
          {{- if .Values.diagnosticMode.enabled }}
          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
          {{- else if .Values.command }}
          command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }}
          {{- else }}
          command:
            - bash
            - -c
            - |
              RANK=${POD_NAME##*-}
              ((RANK++))
              export RANK
              {{- if .Values.entrypoint.file }}
              python {{ .Values.entrypoint.file }} {{ if .Values.entrypoint.args }}{{ .Values.entrypoint.args }}{{ end }}
              {{- end }}
              sleep infinity
          {{- end }}
          {{- if .Values.diagnosticMode.enabled }}
          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
          {{- else if .Values.args }}
          args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }}
          {{- end }}
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: MASTER_ADDR
              value: {{ include "common.names.fullname" . }}
            - name: MASTER_PORT
              value: {{ .Values.containerPorts.pytorch | quote }}
            - name: WORLD_SIZE
              value: {{ .Values.worldSize | quote }}
            {{- if .Values.extraEnvVars }}
            {{- toYaml .Values.extraEnvVars | nindent 12 }}
            {{- end }}
          envFrom:
            {{- if .Values.extraEnvVarsCM }}
            - configMapRef:
                name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsCM "context" $) }}
            {{- end }}
            {{- if .Values.extraEnvVarsSecret }}
            - secretRef:
                name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsSecret "context" $) }}
            {{- end }}
          {{- if .Values.resources }}
          resources: {{- toYaml .Values.resources | nindent 12 }}
          {{- else if ne .Values.resourcesPreset "none" }}
          resources: {{- include "common.resources.preset" (dict "type" .Values.resourcesPreset) | nindent 12 }}
          {{- end }}
          {{- if not .Values.diagnosticMode.enabled }}
          {{- if .Values.customLivenessProbe }}
          livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customLivenessProbe "context" $) | nindent 12 }}
          {{- else if .Values.livenessProbe.enabled }}
          livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }}
            exec:
              command:
                - python
                - -c
                - import torchvision; print(torchvision.__version__)
          {{- end }}
          {{- if .Values.customReadinessProbe }}
          readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customReadinessProbe "context" $) | nindent 12 }}
          {{- else if .Values.readinessProbe.enabled }}
          readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }}
            exec:
              command:
                - python
                - -c
                - import torch; print(torch.__config__.show())
          {{- end }}
          {{- if .Values.customStartupProbe }}
          startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customStartupProbe "context" $) | nindent 12 }}
          {{- else if .Values.startupProbe.enabled }}
          startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }}
            exec:
              command:
                - python
                - -c
                - import torch; print(torch.__config__.show())
          {{- end }}
          {{- end }}
          {{- if .Values.lifecycleHooks }}
          lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.lifecycleHooks "context" $) | nindent 12 }}
          {{- end }}
          volumeMounts:
            {{- if .Values.configMap }}
            - name: ext-files
              mountPath: /app
            {{- else if .Files.Glob "files/*" }}
            - name: local-files
              mountPath: /app
            {{- else if .Values.cloneFilesFromGit.enabled }}
            - name: git-cloned-files
              mountPath: /app
            {{- else }}
            - name: empty-dir
              mountPath: /app
              subPath: app-dir
            {{- end }}
            - name: empty-dir
              mountPath: /tmp
              subPath: tmp-dir
            {{- if .Values.extraVolumeMounts }}
            {{- include "common.tplvalues.render" ( dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
            {{- end }}
            - name: data
              mountPath: {{ .Values.persistence.mountPath }}
      volumes:
        - name: empty-dir
          emptyDir: {}
        {{- if .Values.configMap }}
        - name: ext-files
          configMap:
            name: {{ .Values.configMap }}
        {{- else if .Files.Glob "files/*" }}
        - name: local-files
          configMap:
            name: {{ printf "%s-files" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }}
        {{- end }}
        {{- if .Values.extraVolumes }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.extraVolumes "context" $) | nindent 8 }}
        {{- end }}
  {{- if not .Values.persistence.enabled }}
        - name: data
          emptyDir: {}
  {{- else }}
  volumeClaimTemplates:
    - apiVersion: v1
      kind: PersistentVolumeClaim
      metadata:
        name: data
        {{- if or .Values.commonAnnotations .Values.persistence.annotations }}
        {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.persistence.annotations .Values.commonAnnotations ) "context" . ) }}
        annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 10 }}
        {{- end }}
        {{- if .Values.commonLabels }}
        labels: {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 10 }}
        {{- end }}
      spec:
        accessModes:
        {{- range .Values.persistence.accessModes }}
          - {{ . | quote }}
        {{- end }}
        resources:
          requests:
            storage: {{ .Values.persistence.size | quote }}
        {{- if .Values.persistence.selector }}
        selector: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.selector "context" $) | nindent 10 }}
        {{- end }}
        {{- include "common.storage.class" (dict "persistence" .Values.persistence "global" .Values.global) | nindent 8 }}
  {{- end }}
{{- end }}
