/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.runners.samza;

import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.fnexecution.GrpcFnServer;
import org.apache.beam.runners.fnexecution.ServerFactory;
import org.apache.beam.runners.jobsubmission.InMemoryJobService;
import org.apache.beam.runners.jobsubmission.JobInvocation;
import org.apache.beam.runners.jobsubmission.JobInvoker;
import org.apache.beam.sdk.expansion.service.ExpansionServer;
import org.apache.beam.sdk.expansion.service.ExpansionService;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Driver program that starts a job server. */
// TODO(BEAM-8510): extend JobServerDriver
@SuppressWarnings({
  "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
public class SamzaJobServerDriver {
  private static final Logger LOG = LoggerFactory.getLogger(SamzaJobServerDriver.class);

  private final SamzaPortablePipelineOptions pipelineOptions;

  protected SamzaJobServerDriver(SamzaPortablePipelineOptions pipelineOptions) {
    this.pipelineOptions = pipelineOptions;
  }

  public static void main(String[] args) throws Exception {
    SamzaPortablePipelineOptions pipelineOptions =
        PipelineOptionsFactory.fromArgs(args).as(SamzaPortablePipelineOptions.class);
    fromOptions(pipelineOptions).run();
  }

  public static SamzaJobServerDriver fromOptions(SamzaPortablePipelineOptions pipelineOptions) {
    Map<String, String> overrideConfig =
        pipelineOptions.getConfigOverride() != null
            ? pipelineOptions.getConfigOverride()
            : new HashMap<>();
    overrideConfig.put(SamzaRunnerOverrideConfigs.IS_PORTABLE_MODE, String.valueOf(true));
    overrideConfig.put(
        SamzaRunnerOverrideConfigs.FN_CONTROL_PORT,
        String.valueOf(pipelineOptions.getControlPort()));
    overrideConfig.put(SamzaRunnerOverrideConfigs.FS_TOKEN_PATH, pipelineOptions.getFsTokenPath());

    pipelineOptions.setConfigOverride(overrideConfig);
    return new SamzaJobServerDriver(pipelineOptions);
  }

  private InMemoryJobService createJobService() throws IOException {
    JobInvoker jobInvoker =
        new JobInvoker("samza-job-invoker") {
          @Override
          protected JobInvocation invokeWithExecutor(
              RunnerApi.Pipeline pipeline,
              Struct options,
              @Nullable String retrievalToken,
              ListeningExecutorService executorService)
              throws IOException {
            return new SamzaJobInvocation(pipeline, pipelineOptions);
          }
        };
    return InMemoryJobService.create(
        null,
        session -> session,
        stagingSessionToken -> {},
        jobInvoker,
        InMemoryJobService.DEFAULT_MAX_INVOCATION_HISTORY);
  }

  private ExpansionServer createExpansionService(String host, int expansionPort)
      throws IOException {
    if (host == null) {
      host = InetAddress.getLoopbackAddress().getHostName();
    }
    ExpansionServer expansionServer =
        ExpansionServer.create(new ExpansionService(), host, expansionPort);
    LOG.info(
        "Java ExpansionService started on {}:{}",
        expansionServer.getHost(),
        expansionServer.getPort());
    return expansionServer;
  }

  public void run() throws Exception {
    // Create services
    final InMemoryJobService service = createJobService();
    final GrpcFnServer<InMemoryJobService> jobServiceGrpcFnServer =
        GrpcFnServer.allocatePortAndCreateFor(
            service, ServerFactory.createWithPortSupplier(pipelineOptions::getJobPort));
    final String jobServerUrl = jobServiceGrpcFnServer.getApiServiceDescriptor().getUrl();
    LOG.info("JobServer started on {}", jobServerUrl);
    final URI uri = new URI(jobServerUrl);
    final ExpansionServer expansionServer =
        createExpansionService(uri.getHost(), pipelineOptions.getExpansionPort());

    try {
      jobServiceGrpcFnServer.getServer().awaitTermination();
    } finally {
      LOG.info("JobServer closing");
      jobServiceGrpcFnServer.close();
      if (expansionServer != null) {
        try {
          expansionServer.close();
          LOG.info(
              "Expansion stopped on {}:{}", expansionServer.getHost(), expansionServer.getPort());
        } catch (Exception e) {
          LOG.error("Error while closing the Expansion Service.", e);
        }
      }
    }
  }
}

@@ -961,24 +961,47 @@
 a.util.Map;%0A
+import java.util.UUID;%0A
 import org.a
@@ -1040,16 +1040,93 @@
 nerApi;%0A
+import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;%0A
 import o
@@ -1224,32 +1224,97 @@
 .ServerFactory;%0A
+import org.apache.beam.runners.fnexecution.provisioning.JobInfo;%0A
 import org.apach
@@ -3860,63 +3860,564 @@
     
-return new SamzaJobInvocation(pipeline, pipelineOptions
+SamzaPipelineRunner pipelineRunner = new SamzaPipelineRunner(pipelineOptions);%0A            String invocationId =%0A                String.format(%22%25s_%25s%22, pipelineOptions.getJobName(), UUID.randomUUID().toString());%0A            JobInfo jobInfo =%0A                JobInfo.create(%0A                    invocationId,%0A                    pipelineOptions.getJobName(),%0A                    retrievalToken,%0A                    PipelineOptionsTranslation.toProto(pipelineOptions));%0A            return new JobInvocation(jobInfo, executorService, pipeline, pipelineRunner
 );%0A 

Use SamzaPipelineRunner instead of SamzaJobInvocation to run portable pipelines (#14291)