From d59a8a9676247a078487273278fe77220d67ee66 Mon Sep 17 00:00:00 2001 From: Ladislav Dokoupil Date: Thu, 15 Feb 2024 13:32:12 +0000 Subject: [PATCH] log download on failure --- Azure-README.md | 4 ++ README.md | 5 ++ ...zureArmTemplateCloudDeploymentManager.java | 34 +++++++++- .../java/sunstone/azure/impl/AzureConfig.java | 2 + .../java/sunstone/azure/impl/AzureUtils.java | 65 +++++++++++++++++++ 5 files changed, 108 insertions(+), 2 deletions(-) diff --git a/Azure-README.md b/Azure-README.md index 39daf9f8..47ab4b60 100644 --- a/Azure-README.md +++ b/Azure-README.md @@ -158,3 +158,7 @@ You can deploy archive to running EAP instance. See [here](azure-wildfly/src/tes ###### Web application You can deploy archive to running EAP instance. See [here](azure-wildfly/src/test/java/sunstone/azure/armTemplates/archiveDeploy/webapp/suitetests/AzureWebAppDeployFirstTest.java) +### log downloader [WIP: azure only] + +activityLog Downloader starting Time is configurable by `sunstone.azure.logDownloadTimeStart`, which is offset from current time. Default is 1 hour. +API is restricted by RGName but not specific RG, potentially fetching logs from previous RGs with the same name. diff --git a/README.md b/README.md index 784f956c..e28a0d1f 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ you don't have to do anything. For other loggers, see [the SLF4J manual](http:// The loggers are called `sunstone.*`, short and clear. (For example: `sunstone.core`) +### log downloader [WIP: azure only] + +In the event of a deployment failure, logs are automatically downloaded and stored within the `MODULE/target/logs/*.log` files. +Please note that this feature is currently a work in progress and is available for Azure environments only. + ## License * [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) diff --git a/azure/src/main/java/sunstone/azure/impl/AzureArmTemplateCloudDeploymentManager.java b/azure/src/main/java/sunstone/azure/impl/AzureArmTemplateCloudDeploymentManager.java index dd850ed9..40afa36d 100644 --- a/azure/src/main/java/sunstone/azure/impl/AzureArmTemplateCloudDeploymentManager.java +++ b/azure/src/main/java/sunstone/azure/impl/AzureArmTemplateCloudDeploymentManager.java @@ -2,15 +2,21 @@ import com.azure.core.management.Region; +import com.azure.core.util.polling.LongRunningOperationStatus; +import com.azure.core.util.polling.PollResponse; import com.azure.resourcemanager.AzureResourceManager; +import com.azure.resourcemanager.resources.fluentcore.model.Accepted; +import com.azure.resourcemanager.resources.models.Deployment; import com.azure.resourcemanager.resources.models.DeploymentMode; import com.azure.resourcemanager.resources.models.ResourceGroups; import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.slf4j.Logger; +import sunstone.core.TimeoutUtils; import java.io.IOException; +import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -18,6 +24,7 @@ import java.util.Set; import java.util.UUID; +import static sunstone.core.SunstoneConfig.getValue; /** * Purpose: the class handles Azure template - deploy and undeploy the template to and from a stack. @@ -60,12 +67,35 @@ void deploy(String template, Map parameters, String group, Strin LOGGER.warn("Azure resource group '{}' already exists! It will be reused and deleted when tests are finished.", group); } - armManager.deployments().define(deploymentName) + //.create() doesn't allow status check and fails + Accepted acceptedDeployment = armManager.deployments().define(deploymentName) .withExistingResourceGroup(group) .withTemplate(template) .withParameters(parametersFromMap(template, parameters)) .withMode(DeploymentMode.INCREMENTAL) - .create(); + .beginCreate(); + + // polling wait + final Duration pollInterval = Duration.ofSeconds(TimeoutUtils.adjust(getValue(AzureConfig.DEPLOY_POLL_WAIT, 1))); + LongRunningOperationStatus pollStatus = acceptedDeployment.getActivationResponse().getStatus(); + long delayInMills = pollInterval.toMillis(); + while (!pollStatus.isComplete()) { + try { + Thread.sleep(delayInMills); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + PollResponse pollResponse = acceptedDeployment.getSyncPoller().poll(); + pollStatus = pollResponse.getStatus(); + } + if (pollStatus != LongRunningOperationStatus.SUCCESSFULLY_COMPLETED) { + LOGGER.error("Azure deployment from template {} in \"{}\" group failed", deploymentName, group); + AzureUtils.downloadResourceGroupLogs(armManager, group); + undeploy(group); + throw new RuntimeException("Deployment failed for group:" + group); + } + LOGGER.debug("Azure deployment from template {} in \"{}\" group is ready", deploymentName, group); } diff --git a/azure/src/main/java/sunstone/azure/impl/AzureConfig.java b/azure/src/main/java/sunstone/azure/impl/AzureConfig.java index c2a064b9..9f833da9 100644 --- a/azure/src/main/java/sunstone/azure/impl/AzureConfig.java +++ b/azure/src/main/java/sunstone/azure/impl/AzureConfig.java @@ -10,4 +10,6 @@ public class AzureConfig { public static final String PASSWORD = "sunstone.azure.password"; public static final String REGION = "sunstone.azure.region"; public static final String GROUP = "sunstone.azure.group"; + public static final String DEPLOY_POLL_WAIT = "sunstone.azure.deployPollWait"; + public static final String RG_LOGGER_TIME_START = "sunstone.azure.logDownloadTimeStart"; } diff --git a/azure/src/main/java/sunstone/azure/impl/AzureUtils.java b/azure/src/main/java/sunstone/azure/impl/AzureUtils.java index abcc5529..b3c1dcb0 100644 --- a/azure/src/main/java/sunstone/azure/impl/AzureUtils.java +++ b/azure/src/main/java/sunstone/azure/impl/AzureUtils.java @@ -2,20 +2,34 @@ import com.azure.core.credential.TokenCredential; +import com.azure.core.http.rest.PagedIterable; import com.azure.core.management.AzureEnvironment; +import com.azure.core.management.exception.ManagementError; import com.azure.core.management.profile.AzureProfile; import com.azure.identity.ClientSecretCredentialBuilder; import com.azure.resourcemanager.AzureResourceManager; import com.azure.resourcemanager.appservice.models.AppServicePlan; import com.azure.resourcemanager.appservice.models.WebApp; import com.azure.resourcemanager.compute.models.VirtualMachine; +import com.azure.resourcemanager.monitor.models.EventData; import com.azure.resourcemanager.postgresqlflexibleserver.PostgreSqlManager; import com.azure.resourcemanager.postgresqlflexibleserver.models.Server; +import org.slf4j.Logger; import sunstone.core.SunstoneConfig; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.OffsetDateTime; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; + +import static sunstone.core.SunstoneConfig.getValue; public class AzureUtils { + static Logger LOGGER = AzureLogger.DEFAULT; static AzureResourceManager getResourceManager() { return AzureResourceManager .authenticate(getCredentials(), new AzureProfile(AzureEnvironment.AZURE)) @@ -73,4 +87,55 @@ static Optional findAzurePgSqlServer(PostgreSqlManager pgsqlManager, Str return Optional.empty(); } } + + /** + * API is restricted by RGName but not specific RG, potentially fetching logs from previous RGs with the same name. + */ + static void downloadResourceGroupLogs(AzureResourceManager armManager, String rgName) { + PagedIterable x = armManager.activityLogs().defineQuery() + .startingFrom(OffsetDateTime.now().minusHours(getValue(AzureConfig.RG_LOGGER_TIME_START, 1))) + .endsBefore(OffsetDateTime.now()) + .withAllPropertiesInResponse() + .filterByResourceGroup(rgName) + .execute(); + + Path logDir = Paths.get("logs"); + if (!Files.exists(logDir)) { + try { + Files.createDirectories(logDir); + } catch (IOException e) { + LOGGER.error("Error creating log directory", e); + return; + } + } + + Path activityLogFile = logDir.resolve(rgName + "-activity-log.log"); + try(java.io.FileWriter writer = new java.io.FileWriter(activityLogFile.toString())) { + for (EventData e : x) { + writer.write(e.operationName().localizedValue() + "\tname: " + e.eventName().localizedValue() + "\tprops: " + e.properties() + "\tdesc: " + e.description() + "\ttimestamp: " + e.eventTimestamp() + "\tlevel: " + e.level() + "\n\n"); + } + } catch (IOException e) { + LOGGER.error("Error writing activity log", e); + } + + Path deploymentLogFile = logDir.resolve(rgName + "-deployments.log"); + try(java.io.FileWriter writer = new java.io.FileWriter(deploymentLogFile.toString())) { + armManager.deployments().listByResourceGroup(rgName).forEach(d -> { + List operations = d.deploymentOperations().list().stream().map(s -> "\n" + s.timestamp() + ", state: " + s.provisioningState() + ", op: " + s.provisioningOperation() + ", resource: " + (s.targetResource() != null ? s.targetResource().resourceName() : "UNKNOWN_NAME") + ", type: " + (s.targetResource() != null ? s.targetResource().resourceType() : "UNKNOWN_TYPE")).collect(Collectors.toList()); + try { + writer.write(d.error().getMessage()); + writer.write("\nError message: \n"); + writer.write(d.error().getDetails().stream().map(ManagementError::getMessage).collect(Collectors.joining())); + writer.write("\nDeployment Info:\n"); + writer.write(d.name() + "\t" + d.provisioningState() + "\t" + d.timestamp() + "\twith:" + operations + "\n"); + writer.write("\n\n"); + } catch (IOException e) { + LOGGER.error("Error writing deployment log", e); + } + }); + } catch (IOException e) { + LOGGER.error("Error writing deployment log", e); + } + } + }