robbond/pedwfrontend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added container and blob sas tokens

Browse Source
This commit is contained in:
robbond
2022-10-25 14:00:46 +01:00
parent 4e234ef756
commit c99205d4e9
3 changed files with 168 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
actions/azurestorage.js
+148 -116
View File
@@ -1,15 +1,115 @@
import { v4 as uuidv4 } from "uuid";
import { BlobServiceClient, ContainerClient } from "@azure/storage-blob";
import {
DefaultAzureCredential,
InteractiveBrowserCredential,
EnvironmentCredential,
ClientSecretCredential,
} from "@azure/identity";
import { consoleLogger, hashAPIPath } from ".";
ContainerClient,
BlockBlobClient,
BlobServiceClient,
BlobSASPermissions,
ContainerSASPermissions,
generateBlobSASQueryParameters,
SASProtocol,
} from "@azure/storage-blob";
import { DefaultAzureCredential } from "@azure/identity";
import { consoleLogger, conLog, hashAPIPath } from ".";
import _ from "lodash";
const STORAGE_PATH = process.env.AZURE_PEDW_STORAGE_ENDPOINT;
const STORAGE_CONTAINER = process.env.AZURE_PEDW_CONTAINER;
const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME;
export const createContainerSas = async (containerName) => {
// Get environment variables
// Best practice: create time limits
const TEN_MINUTES = 10 * 60 * 1000;
const NOW = new Date();
// Best practice: set start time a little before current time to
// make sure any clock issues are avoided
const TEN_MINUTES_BEFORE_NOW = new Date(NOW.valueOf() - TEN_MINUTES);
const TEN_MINUTES_AFTER_NOW = new Date(NOW.valueOf() + TEN_MINUTES);
// Best practice: use managed identity - DefaultAzureCredential
const blobServiceClient = new BlobServiceClient(
`${STORAGE_PATH}`,
new DefaultAzureCredential()
);
// Best practice: delegation key is time-limited
// When using a user delegation key, container must already exist
const userDelegationKey = await blobServiceClient.getUserDelegationKey(
TEN_MINUTES_BEFORE_NOW,
TEN_MINUTES_AFTER_NOW
);
// Need only list permission to list blobs
const containerPermissions = "rcwltd";
// Best practice: SAS options are time-limited
const sasOptions = {
containerName,
permissions: ContainerSASPermissions.parse(containerPermissions),
protocol: SASProtocol.HttpsAndHttp,
startsOn: TEN_MINUTES_BEFORE_NOW,
expiresOn: TEN_MINUTES_AFTER_NOW,
};
//conLogJSON.stringify(sasOptions));
const sasToken = generateBlobSASQueryParameters(
sasOptions,
userDelegationKey,
accountName
).toString();
return sasToken;
};
export const createBlobSas = async (containerName, blobName) => {
// Get environment variables
const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME;
// Best practice: create time limits
const TEN_MINUTES = 10 * 60 * 1000;
const NOW = new Date();
// Best practice: set start time a little before current time to
// make sure any clock issues are avoided
const TEN_MINUTES_BEFORE_NOW = new Date(NOW.valueOf() - TEN_MINUTES);
const TEN_MINUTES_AFTER_NOW = new Date(NOW.valueOf() + TEN_MINUTES);
// Best practice: use managed identity - DefaultAzureCredential
const blobServiceClient = new BlobServiceClient(
`https://${accountName}.blob.core.windows.net`,
new DefaultAzureCredential()
);
// Best practice: delegation key is time-limited
// When using a user delegation key, container must already exist
const userDelegationKey = await blobServiceClient.getUserDelegationKey(
TEN_MINUTES_BEFORE_NOW,
TEN_MINUTES_AFTER_NOW
);
// Need only create/write permission to upload file
const blobPermissionsForAnonymousUser = "rcwt";
// Best practice: SAS options are time-limited
const sasOptions = {
blobName,
containerName,
permissions: BlobSASPermissions.parse(blobPermissionsForAnonymousUser),
protocol: SASProtocol.HttpsAndHttp,
startsOn: TEN_MINUTES_BEFORE_NOW,
expiresOn: TEN_MINUTES_AFTER_NOW,
};
const sasToken = generateBlobSASQueryParameters(
sasOptions,
userDelegationKey,
accountName
).toString();
return sasToken;
};
export const createContainer = async (containerName) => {
const creds = new DefaultAzureCredential();
@@ -30,15 +130,6 @@ export const createContainer = async (containerName) => {
const blobServiceClient = new BlobServiceClient(`${STORAGE_PATH}`, creds);
const createContainerResponse = await containerClient.createIfNotExists();
// console.log(
// `Created container ${containerName} successfully`,
// createContainerResponse.requestId
// );
// console.log("Containers:");
// for await (const container of blobServiceClient.listContainers()) {
// console.log(`- ${container.name}`);
// }
console.log(
"\n//////////////////\n container name :",
@@ -60,14 +151,11 @@ export const getContainers = async () => {
};
export const getBlobs = async (containerName, casefolderID) => {
const creds = new DefaultAzureCredential();
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
containerClient.createIfNotExists();
//conLog"getBlobs " + sasUrl);
const blobObj = [];
for await (const blob of containerClient.listBlobsFlat({
@@ -131,12 +219,9 @@ export const getBlobs = async (containerName, casefolderID) => {
};
export const createBlob = async (formContent, containerName, caseref) => {
const creds = new DefaultAzureCredential();
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
formContent = JSON.parse(formContent);
let caseID = formContent.pinswg_name;
@@ -161,14 +246,13 @@ export const deleteBlob = async (containerName, blobName) => {
deleteSnapshots: "include", // or 'only'
};
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName.toLowerCase()}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
await blockBlobClient.deleteIfExists(options);
await blockBlobClient.delete(options);
console.log(`deleted blob ${blobName}`);
@@ -182,10 +266,11 @@ export const deleteBlobCase = async (containerName, blobName) => {
deleteSnapshots: "include", // or 'only'
};
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
//conLog"deleteBlobCase ");
console.log("blob to delete:", blobName);
@@ -205,12 +290,9 @@ export const deleteBlobCase = async (containerName, blobName) => {
};
export const uploadFile = async (formContent, containerName, foldername) => {
const creds = new DefaultAzureCredential();
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
const files = formContent;
@@ -236,13 +318,9 @@ export const uploadFile = async (formContent, containerName, foldername) => {
export const downloadFile = async (containerName, blobName) => {
//console.log.apply(containerName, blobName);
const creds = new DefaultAzureCredential();
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
const blobClient = containerClient.getBlobClient(blobName);
const downloadedBlob = await blobClient.download(0);
@@ -256,13 +334,10 @@ export const downloadProgressFile = async (
casefolderID
) => {
//console.log.apply(containerName, blobName);
const creds = new DefaultAzureCredential();
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
const blobClient = containerClient.getBlobClient(blobName);
const downloadedBlob = await blobClient.download(0);
@@ -276,15 +351,13 @@ export const downloadAllProgressFiles = async (
) => {
console.log(
"/////////////////////////\n downloading files: " +
progressBlobObj +
JSON.stringify(progressBlobObj) +
"\n/////////////////////////\n"
);
const creds = new DefaultAzureCredential();
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const containerToken = await createContainerSas(containerName);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
const containerClient = new ContainerClient(sasUrl);
let blobClient = {};
let downloadedBlob = {};
@@ -349,45 +422,28 @@ export const getCaseBlob = async (
caseReference,
formContent
) => {
const creds = new DefaultAzureCredential();
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
containerClient.createIfNotExists();
let blobCount = 0;
for await (const blob of containerClient.listBlobsFlat({
prefix: caseReference,
})) {
blobCount++;
}
const content = JSON.stringify(formContent);
const blobName = caseReference + "/case/" + caseReference + "_case.json";
console.log("blobName:", blobName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const containerBlobToken = await createBlobSas(containerName, blobName);
const blobSasUrl = `${STORAGE_PATH}/${containerName}/${blobName}?${containerBlobToken}`;
const blockBlobClient = new BlockBlobClient(blobSasUrl);
const uploadBlobResponse = await blockBlobClient.upload(
content,
Buffer.byteLength(content)
);
return blobName;
};
export const getProgressBlobs = async (containerName, caseReference) => {
const creds = new DefaultAzureCredential();
const containerToken = await createContainerSas(containerName);
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
containerClient.createIfNotExists();
const containerClient = new ContainerClient(sasUrl);
let blobCount = 0;
for await (const blob of containerClient.listBlobsFlat({
@@ -449,28 +505,16 @@ export const getProgressBlobs = async (containerName, caseReference) => {
};
export const getAllProgressBlobs = async (containerName) => {
const creds = new DefaultAzureCredential();
const containerToken = await createContainerSas(containerName);
const containerClient = new ContainerClient(
`${STORAGE_PATH}/${containerName}`,
creds
);
const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`;
containerClient.createIfNotExists();
const containerClient = new ContainerClient(sasUrl);
let blobCount = 0;
for await (const blob of containerClient.listBlobsFlat({
// prefix: caseReference,
})) {
blobCount++;
}
console.log("this is the caasefolder:", blobCount);
let blobObj = [];
for await (const blob of containerClient.listBlobsFlat({
//prefix: caseReference + "/" + caseReference + ".json",
})) {
for await (const blob of containerClient.listBlobsFlat()) {
blob.name.split("/")[1].indexOf(".json") > 0 &&
blob.name.split("/")[1].indexOf("undefined") < 0 &&
blobObj.push({
@@ -486,18 +530,6 @@ export const getAllProgressBlobs = async (containerName) => {
"contentLength": blob.properties.contentLength,
"contentType": blob.contentType,
"lastModified": blob.properties.lastModified,
// "hashedfilepath": hashAPIPath(
// "/api/file/downloadblob?container=" +
// containerName +
// ),
// "hasheddeletepath": hashAPIPath(
// "/api/file/deleteblob?container=" +
// containerName +
// ),
// "hashgetblobs": hashAPIPath(
// "/api/file/getbloblist?container=" +
// containerName +
// ),
});
}