We can use REST API method for processing AAS cube , please find the full adf pipeline below which capture the logging as well if error comes up .REST API can processing only full data for AAS cube.
This pipeline is very generic which we can use for minimum effort of code changes you just need to give client secret name on global variable . hence your secret code gets encrypted .
Pre Requisition :-APP registration ,ADF, Grant access to AAS server to APPs .
{
"name": "PL_AAS_Cube_Processing",
"properties": {
"activities": [
{
"name": "UntilRefreshComplete",
"type": "Until",
"dependsOn": [
{
"activity": "FilterToCurrentRefresh",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"expression": {
"value": "@not(equals(activity('GetAzureASRefreshStatus').output.status,'inProgress'))\n\n\n\n\n",
"type": "Expression"
},
"activities": [
{
"name": "GetAzureASRefreshStatus",
"type": "WebActivity",
"dependsOn": [
{
"activity": "Wait30Seconds",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.Region,'.asazure.windows.net/servers/',pipeline().parameters.Server,'/models/',pipeline().parameters.DatabaseName,'/refreshes/',activity('FilterToCurrentRefresh').output.Value[0].refreshId)",
"type": "Expression"
},
"method": "GET",
"headers": {
"Authorization": {
"value": "@concat(pipeline().parameters.Token_type,' ',pipeline().parameters.Access_token)",
"type": "Expression"
}
},
"authentication": {
"type": "MSI",
"resource": "https://*.asazure.windows.net"
}
}
},
{
"name": "Wait30Seconds",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 30
}
}
],
"timeout": "7.00:00:00"
}
},
{
"name": "FilterToCurrentRefresh",
"type": "Filter",
"dependsOn": [
{
"activity": "GetAzureASRefreshes",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@json(activity('GetAzureASRefreshes').output.Response)",
"type": "Expression"
},
"condition": {
"value": "@greaterOrEquals(item().startTime,addseconds(activity('StartProcessingAzureAS').output.startTime,-30))",
"type": "Expression"
}
}
},
{
"name": "StartProcessingAzureAS",
"type": "WebActivity",
"dependsOn": [
{
"activity": "DeleteAlreadyRunningApiCall",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.Region,'.asazure.windows.net/servers/',pipeline().parameters.Server,'/models/',pipeline().parameters.DatabaseName,'/refreshes')",
"type": "Expression"
},
"method": "POST",
"headers": {
"Authorization": {
"value": "@concat(pipeline().parameters.Token_type,' ',pipeline().parameters.Access_token)",
"type": "Expression"
}
},
"body": {
"value": "@pipeline().parameters.MsgBody",
"type": "Expression"
},
"disableCertValidation": false
}
},
{
"name": "IfFailed",
"type": "IfCondition",
"dependsOn": [
{
"activity": "UntilRefreshComplete",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"expression": {
"value": "@equals(activity('GetAzureASRefreshStatus').output.status,'failed')",
"type": "Expression"
},
"ifTrueActivities": [
{
"name": "ThrowErrorOnFailure",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@string(activity('GetAzureASRefreshStatus').output)",
"type": "Expression"
},
"method": "GET"
}
}
]
}
},
{
"name": "GetAzureASRefreshes",
"type": "WebActivity",
"dependsOn": [
{
"activity": "StartProcessingAzureAS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 5,
"retryIntervalInSeconds": 30,
"secureOutput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.Region,'.asazure.windows.net/servers/',pipeline().parameters.Server,'/models/',pipeline().parameters.DatabaseName,'/refreshes')",
"type": "Expression"
},
"method": "GET",
"headers": {
"Authorization": {
"value": "@concat(pipeline().parameters.Token_type,' ',pipeline().parameters.Access_token)",
"type": "Expression"
}
},
"body": {
"Type": "Full",
"CommitMode": "transactional",
"MaxParallelism": 10,
"RetryCount": 2
},
"disableCertValidation": false
}
},
{
"name": "SuccessOnDimLoading",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "IfFailed",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[STAGING].[usp_InsertLoggingRecords]",
"storedProcedureParameters": {
"ACTIVITY_NAME": {
"value": {
"value": "@concat(pipeline().parameters.DatabaseName,' - Data Processing')",
"type": "Expression"
},
"type": "String"
},
"ERROR": {
"value": null,
"type": "String"
},
"FILEPATH": {
"value": null,
"type": "String"
},
"PIPELINE_NAME": {
"value": {
"value": "@pipeline().Pipeline",
"type": "Expression"
},
"type": "String"
},
"ROWS_READ": {
"value": "1",
"type": "Int32"
},
"ROWS_WRITTEN": {
"value": null,
"type": "Int32"
},
"STAGE": {
"value": "FullDataLoad",
"type": "String"
},
"START_DATE": {
"value": {
"value": "@variables('StartDateDim')",
"type": "Expression"
},
"type": "Datetime"
},
"STATUS": {
"value": "SUCCESS",
"type": "String"
},
"TABLE_NAME": {
"value": {
"value": "@pipeline().parameters.TableName",
"type": "Expression"
},
"type": "String"
}
}
},
"linkedServiceName": {
"referenceName": "Connection_SQLDB_AzureIR",
"type": "LinkedServiceReference"
}
},
{
"name": "FailureOnDimLoading",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "IfFailed",
"dependencyConditions": [
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[STAGING].[usp_InsertLoggingRecords]",
"storedProcedureParameters": {
"ACTIVITY_NAME": {
"value": {
"value": "@concat(pipeline().parameters.DatabaseName,' - Data Processing')",
"type": "Expression"
},
"type": "String"
},
"ERROR": {
"value": {
"value": "@activity('IfFailed').Error.message",
"type": "Expression"
},
"type": "String"
},
"FILEPATH": {
"value": null,
"type": "String"
},
"PIPELINE_NAME": {
"value": {
"value": "@pipeline().Pipeline",
"type": "Expression"
},
"type": "String"
},
"ROWS_READ": {
"value": "1",
"type": "Int32"
},
"ROWS_WRITTEN": {
"value": null,
"type": "Int32"
},
"STAGE": {
"value": {
"value": "FullDataLoad",
"type": "Expression"
},
"type": "String"
},
"START_DATE": {
"value": {
"value": "@variables('StartDateDim')",
"type": "Expression"
},
"type": "DateTime"
},
"STATUS": {
"value": "FAILED",
"type": "String"
},
"TABLE_NAME": {
"value": {
"value": "@pipeline().parameters.TableName",
"type": "Expression"
},
"type": "String"
}
}
},
"linkedServiceName": {
"referenceName": "Connection_SQLDB_AzureIR",
"type": "LinkedServiceReference"
}
},
{
"name": "SetvariableStartDate",
"type": "SetVariable",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"variableName": "StartDateDim",
"value": {
"value": "@utcnow()",
"type": "Expression"
}
}
},
{
"name": "DeleteAlreadyRunningApiCall",
"type": "WebActivity",
"dependsOn": [
{
"activity": "FilterToCurrentRefreshForDelete",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.Region,'.asazure.windows.net/servers/',pipeline().parameters.Server,'/models/',pipeline().parameters.DatabaseName,'/refreshes/',activity('FilterToCurrentRefreshForDelete').output.Value[0].refreshId)",
"type": "Expression"
},
"method": "DELETE",
"headers": {
"Authorization": {
"value": "@concat(pipeline().parameters.Token_type,' ',pipeline().parameters.Access_token)",
"type": "Expression"
}
}
}
},
{
"name": "GetAzureASRefreshesForDelete",
"type": "WebActivity",
"dependsOn": [
{
"activity": "SetvariableStartDate",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 5,
"retryIntervalInSeconds": 30,
"secureOutput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.Region,'.asazure.windows.net/servers/',pipeline().parameters.Server,'/models/',pipeline().parameters.DatabaseName,'/refreshes')",
"type": "Expression"
},
"method": "GET",
"headers": {
"Authorization": {
"value": "@concat(pipeline().parameters.Token_type,' ',pipeline().parameters.Access_token)",
"type": "Expression"
}
},
"body": {
"Type": "Full",
"CommitMode": "transactional",
"MaxParallelism": 10,
"RetryCount": 2
},
"disableCertValidation": false
}
},
{
"name": "FilterToCurrentRefreshForDelete",
"type": "Filter",
"dependsOn": [
{
"activity": "GetAzureASRefreshesForDelete",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@json(activity('GetAzureASRefreshesForDelete').output.Response)",
"type": "Expression"
},
"condition": {
"value": "@greaterOrEquals(100,10)",
"type": "Expression"
}
}
}
],
"parameters": {
"TenantID": {
"type": "String",
"defaultValue": ""
},
"ClientID": {
"type": "String",
"defaultValue": ""
},
"SubscriptionID": {
"type": "String",
"defaultValue": ""
},
"Region": {
"type": "String",
"defaultValue": "eastus"
},
"Server": {
"type": "String",
"defaultValue": "preeudsoaanlys01"
},
"DatabaseName": {
"type": "String",
"defaultValue": "CCB_FA"
},
"Token_type": {
"type": "string"
},
"Access_token": {
"type": "string"
},
"TableName": {
"type": "string"
},
"MsgBody": {
"type": "string"
}
},
"variables": {
"StartDateDim": {
"type": "String"
}
},
"folder": {
"name": "Azure AS"
},
"annotations": [],
"lastPublishTime": "2021-09-09T10:35:22Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}
Above json code you can paste into adf pipeline please keep same pipeline name for else you gets error while creating pipeline .
