import { DefaultAzureCredential } from "@azure/identity"; import _ from "lodash"; import { hashAPIPath, consoleLogger } from "."; import { getDocumentTypeFromFilename } from "../components/utils"; const { ContainerClient, BlockBlobClient, BlobServiceClient, BlobSASPermissions, ContainerSASPermissions, generateBlobSASQueryParameters, SASProtocol, } = require("@azure/storage-blob"); const { QueueServiceClient, AccountSASResourceTypes, AccountSASServices, AccountSASPermissions, QueueSASPermissions, QueueSASSignatureValues, generateAccountSASQueryParameters, generateQueueSASQueryParameters, StorageSharedKeyCredential, QueueClient, } = require("@azure/storage-queue"); const STORAGE_PATH = process.env.AZURE_PEDW_STORAGE_ENDPOINT; const STORAGE_CONTAINER = process.env.AZURE_PEDW_CONTAINER; const QUEUE_PATH = process.env.AZURE_PEDW_QUEUE_ENDPOINT; const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME; export const createContainerSas = async (containerName) => { // Get environment variables containerName.length > 0; // Best practice: create time limits const TEN_MINUTES = 10 * 60 * 1000; const NOW = new Date(); // Best practice: set start time a little before current time to // make sure any clock issues are avoided const TEN_MINUTES_BEFORE_NOW = new Date(NOW.valueOf() - TEN_MINUTES); const TEN_MINUTES_AFTER_NOW = new Date(NOW.valueOf() + TEN_MINUTES); // Best practice: use managed identity - DefaultAzureCredential const blobServiceClient = new BlobServiceClient( `${STORAGE_PATH}`, new DefaultAzureCredential() ); // Best practice: delegation key is time-limited // When using a user delegation key, container must already exist const userDelegationKey = await blobServiceClient.getUserDelegationKey( TEN_MINUTES_BEFORE_NOW, TEN_MINUTES_AFTER_NOW ); // Need only list permission to list blobs const containerPermissions = "rcwdltf"; // Best practice: SAS options are time-limited const sasOptions = { containerName, permissions: ContainerSASPermissions.parse(containerPermissions), protocol: SASProtocol.HttpsAndHttp, startsOn: TEN_MINUTES_BEFORE_NOW, expiresOn: TEN_MINUTES_AFTER_NOW, }; // console.log( // "\n////////////////////////\nsasOptions :", // JSON.stringify(sasOptions), // "\n////////////////////////" // ); const sasToken = generateBlobSASQueryParameters( sasOptions, userDelegationKey, accountName ).toString(); // console.log( // "\n////////////////////////\nsasToken :", // sasToken, // "\n////////////////////////" // ); return sasToken; }; export const createBlobSas = async (containerName, blobName) => { // Get environment variables const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME; // Best practice: create time limits const TEN_MINUTES = 10 * 60 * 1000; const NOW = new Date(); // Best practice: set start time a little before current time to // make sure any clock issues are avoided const TEN_MINUTES_BEFORE_NOW = new Date(NOW.valueOf() - TEN_MINUTES); const TEN_MINUTES_AFTER_NOW = new Date(NOW.valueOf() + TEN_MINUTES); // Best practice: use managed identity - DefaultAzureCredential const blobServiceClient = new BlobServiceClient( `https://${accountName}.blob.core.windows.net`, new DefaultAzureCredential() ); // Best practice: delegation key is time-limited // When using a user delegation key, container must already exist const userDelegationKey = await blobServiceClient.getUserDelegationKey( TEN_MINUTES_BEFORE_NOW, TEN_MINUTES_AFTER_NOW ); // Need only create/write permission to upload file const blobPermissionsForAnonymousUser = "rcwt"; // Best practice: SAS options are time-limited const sasOptions = { blobName, containerName, permissions: BlobSASPermissions.parse(blobPermissionsForAnonymousUser), protocol: SASProtocol.HttpsAndHttp, startsOn: TEN_MINUTES_BEFORE_NOW, expiresOn: TEN_MINUTES_AFTER_NOW, }; const sasToken = generateBlobSASQueryParameters( sasOptions, userDelegationKey, accountName ).toString(); return sasToken; }; export const createContainer = async (containerName) => { const creds = new DefaultAzureCredential(); //console.log(JSON.stringify(creds)); //containerName = containerName.toLowerCase(); //console.log( // "container name:", // containerName, // STORAGE_PATH + "/" + containerName // ); const containerClient = new ContainerClient( `${STORAGE_PATH}/${containerName}`, creds ); const blobServiceClient = new BlobServiceClient(`${STORAGE_PATH}`, creds); const createContainerResponse = await containerClient .createIfNotExists() .then((data) => { //console.log(data); return data; }) .catch((error) => { consoleLogger(error); }); return createContainerResponse; }; export const getContainers = async () => { const creds = new DefaultAzureCredential(); const blobServiceClient = new BlobServiceClient(`${STORAGE_PATH}`, creds); //console.log("Containers:"); for await (const container of blobServiceClient.listContainers()) { console.log(`- ${container.name}`); } }; export const getBlobs = async (containerName, casefolderID) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); //conLog"getBlobs " + sasUrl); const blobObj = []; for await (const blob of containerClient.listBlobsFlat({ prefix: casefolderID + "/files/", })) { let blobDocumentType = blob.name .split("/")[2] .slice(0, blob.name.split("/")[2].indexOf("_")); blobObj.push({ "name": blob.name.split("/")[2], "path": blob.name, "documentType": getDocumentTypeFromFilename( blob.name.split("/")[2] ), "versionId": blob.versionId, "caseObj": casefolderID + "/" + casefolderID + "_case.json", "isCurrentVersion": blob.isCurrentVersion, "contentLength": blob.properties.contentLength, "size": blob.properties.contentLength, "contentType": blob.contentType, "lastModified": blob.properties.lastModified, "filepath": "/api/file/downloadblob?container=" + containerName + "&casefolderID=" + encodeURIComponent(casefolderID) + "&blobname=" + encodeURIComponent(blob.name), "hashedfilepath": hashAPIPath( "/api/file/downloadblob?container=" + containerName + "&casefolderID=" + encodeURIComponent(casefolderID) + "&blobname=" + encodeURIComponent(blob.name.split("/")[2]) ), "deletepath": "/api/file/deleteblob?container=" + containerName + "&casefolderID=" + encodeURIComponent(casefolderID) + "&blobname=" + encodeURIComponent(blob.name.split("/")[2]), "hasheddeletepath": hashAPIPath( "/api/file/deleteblob?container=" + containerName + "&casefolderID=" + encodeURIComponent(casefolderID) + "&blobname=" + encodeURIComponent(blob.name.split("/")[2]) ), "hashgetblobs": hashAPIPath( "/api/file/getbloblist?container=" + containerName + "&casefolderID=" + casefolderID ), }); } //console.log("blobObj:", blobObj); return blobObj; }; export const createBlob = async (formContent, containerName, caseref) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); formContent = JSON.parse(formContent); let caseID = ""; caseID = _.has(formContent, "pinswg_name") ? formContent.pinswg_name : caseref; const content = JSON.stringify(formContent); console.log(content); const blobName = caseID + "/" + caseID + "_appeal.json"; console.log("blobName:", blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.upload( content, Buffer.byteLength(content) ); const tags = { containerid: containerName, caseID: caseID, blobType: "Appeal", }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); return formContent.pinswg_name; }; export const createRepBlob = async (formContent, containerName, caseref) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); formContent = JSON.parse(formContent); const content = JSON.stringify(formContent); //console.log(content); const blobName = caseref + "/" + formContent.repfile_name + "_rep.json"; console.log("blobName:", blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.upload( content, Buffer.byteLength(content) ); const tags = { containerid: containerName, caseID: formContent.ticketnumber, blobType: "Representation", }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); return formContent.ticketnumber; }; export const createAppealPDFBlob = async ( formContent, containerName, caseref ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); formContent = formContent; let caseID = ""; caseID = caseref; const content = formContent; const repDate = new Date(); let day = repDate.getDate(); let month = repDate.getMonth() + 1; let year = repDate.getFullYear(); const pdfFileName = year + "-" + ("0" + month).slice(-2) + "-" + ("0" + day).slice(-2) + "_-_Appeal_Form"; //console.log(content); const blobName = caseID + "/files/" + pdfFileName + ".pdf"; const uploadOptions = { blockSize: 4 * 1024 * 1024, // 4 MiB max block size concurrency: 2, // maximum number of parallel transfer workers maxSingleShotSize: 8 * 1024 * 1024, // 8 MiB initial transfer size }; console.log("blobName:", blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadStream(content); console.log(uploadBlobResponse); let whichLocation = "846040000"; // Default value const tags = { containerid: containerName, caseID: caseID, blobType: "Appeal PDF", ishareLocation: whichLocation, documentLocationId: whichLocation, }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); //return formContent.pinswg_name; return uploadBlobResponse; }; export const createRepPDFBlob = async ( formContent, containerName, caseref, repName ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); formContent = formContent; let caseID = ""; caseID = caseref; const content = formContent; //console.log(content); const blobName = caseID + "/" + repName + "/files/" + repName + ".pdf"; const uploadOptions = { blockSize: 4 * 1024 * 1024, // 4 MiB max block size concurrency: 2, // maximum number of parallel transfer workers maxSingleShotSize: 8 * 1024 * 1024, // 8 MiB initial transfer size }; console.log("blobName:", blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadStream(content); let whichLocation = repName.indexOf("_IP_") > 0 ? "846040005" : repName.indexOf("_Statement_") > 0 ? "846040002" : repName.indexOf("_Questionnaire_") > 0 ? "846040001" : repName.indexOf("_Comments") > 0 ? "846040003" : repName.indexOf("_Impact") > 0 ? "846040001" : "846040002"; const tags = { containerid: containerName, caseID: caseID, blobType: "Representation PDF", ishareLocation: whichLocation, documentLocationId: whichLocation, }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); return uploadBlobResponse; }; export const deleteBlob = async (containerName, blobName) => { const creds = new DefaultAzureCredential(); const options = { deleteSnapshots: "include", // or 'only' }; const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const blockBlobClient = containerClient.getBlockBlobClient(blobName); await blockBlobClient.delete(options); console.log(`deleted blob ${blobName}`); return { "deleted": blobName }; }; export const deleteBlobCase = async (containerName, blobName) => { const creds = new DefaultAzureCredential(); const options = { deleteSnapshots: "include", // or 'only' }; const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); //conLog"deleteBlobCase "); console.log("blob to delete:", blobName); for await (const blob of containerClient.listBlobsFlat({ prefix: blobName, })) { console.log(" ------ :", blob.name); containerClient.deleteBlob(blob.name); } //containerClient.deleteBlob(blobName); // console.log(`deleted blob ${blobName}`); // for await (const blob of containerClient.listBlobsFlat()) { // console.log(" ------ :", blob.name); // } return { "deleted": blobName }; }; export const deleteBlobRep = async (containerName, blobName) => { const creds = new DefaultAzureCredential(); const options = { deleteSnapshots: "include", // or 'only' }; const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); console.log("blob to delete:", blobName); for await (const blob of containerClient.listBlobsFlat({ prefix: blobName, })) { console.log("------ :", blob.name); const blockBlobClient = containerClient.getBlockBlobClient(blob.name); console.log("=======:", await blockBlobClient.exists()); (await blockBlobClient.exists()) && containerClient.deleteBlob(blob.name); //containerClient.deleteIfExists(); //containerClient.deleteBlob(blob.name); } //containerClient.deleteBlob(blobName); //console.log(`deleted blob ${blobName}`); // for await (const blob of containerClient.listBlobsFlat()) { // console.log(" ------ :", blob.name); // } return { "deleted": blobName }; }; export const uploadFile = async (formContent, containerName, foldername) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const files = formContent; let blobResponseArr = []; //console.log(...formContent); console.log("files...", files, Object.keys(files).length, foldername); for (const prop in files) { console.log(`files[${prop}] = ${files[prop][0].size}`); const blobName = foldername + "/files/" + files[prop][0].fieldName; console.log(blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient .uploadFile(files[prop][0].path, files[prop].size) .then((data) => { console.log( "////////////////////////\nfile name:", files[prop][0].fieldName, "\n////////////////////////////" ); console.log(data); let whichLocation = files[prop][0].fieldName.indexOf("_Statement_of_Case") > 0 || files[prop][0].fieldName.indexOf("_-_Statement_of_Case") > 0 || files[prop][0].fieldName.indexOf("_-_Application_Form") > 0 || files[prop][0].fieldName.indexOf( "_-_Site_Ownership_Certificate" ) > 0 || files[prop][0].fieldName.indexOf("_-_Decision_Notice") > 0 || files[prop][0].fieldName.indexOf("_-_Site_Location_Plan") > 0 || files[prop][0].fieldName.indexOf( "_-_Plans_Drawing_Documents" ) > 0 || files[prop][0].fieldName.indexOf( "_-_Additional_Plans_Drawings_Documents" ) > 0 || files[prop][0].fieldName.indexOf( "_-_Design_and_Access_Statement" ) > 0 || files[prop][0].fieldName.indexOf( "_-_NSB_LPA_Additional_Documents" ) > 0 || files[prop][0].fieldName.indexOf("_-_LPA_Correspondence") > 0 || files[prop][0].fieldName.indexOf( "_-_LPA_Original_Permission" ) > 0 || files[prop][0].fieldName.indexOf( "_-_LPA's_Registration_Letter" ) > 0 || files[prop][0].fieldName.indexOf( +"_-_Environmental_Statement" ) > 0 || files[prop][0].fieldName.indexOf("_-_Cost_of_Application") > 0 || files[prop][0].fieldName.indexOf( "_-_Other_Relevant_Material" ) > 0 || files[prop][0].fieldName.indexOf( "_-_S106_Agreement_or_Unilateral_Undertaking" > 0 ) ? "846040000" : files[prop][0].fieldName.indexOf("_IP_") > 0 ? "846040005" : files[prop][0].fieldName.indexOf("_Statement_") > 0 ? "846040002" : files[prop][0].fieldName.indexOf("_Questionnaire_") > 0 ? "846040001" : files[prop][0].fieldName.indexOf("_Comments_") > 0 ? "846040003" : files[prop][0].fieldName.indexOf("_Impact_") > 0 ? "846040001" : "846040000"; const tags = { containerid: containerName, caseID: foldername.split("/")[0], documentType: files[prop][0].fieldName.split(".")[1], blobType: "RepresentationFile", ishareLocation: whichLocation, documentLocationId: whichLocation, }; const withTags = blockBlobClient.setTags(tags); const withMeta = blockBlobClient.setMetadata(tags); console.log( `Uploaded block blob ${files[prop][0].fieldName} successfully` //uploadBlobResponse.requestId ); blobResponseArr.push({ file: files[prop][0].fieldName }); }); } return blobResponseArr; }; export const uploadSingleFile = async ( formContent, containerName, foldername ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const files = formContent; //console.log(...formContent); console.log("files...", files, Object.keys(files).length, foldername); let blobResponseArr = []; for (const prop in files) { console.log(`files[${prop}] = ${files[prop][0].size}`); const blobName = foldername + "/files/" + files[prop][0].fieldName; console.log(blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadFile( files[prop][0].path, files[prop].size // { // blobHTTPHeaders: { // blobContentType: "application/octet-stream", // }, // onProgress: (progress) => { // // Log upload progress, you can emit this back to the client // console.log( // `Progress: ${progress.loadedBytes} bytes uploaded` // ); // }, // } ); console.log( "////////////////////////\nfile name:", files[prop][0].fieldName, "\n////////////////////////////" ); // let whichLocation = // files[prop][0].fieldName.indexOf("_Statement_of_Case") > 0 || // files[prop][0].fieldName.indexOf("_-_Statement_of_Case") > 0 || // files[prop][0].fieldName.indexOf("_-_Application_Form") > 0 || // files[prop][0].fieldName.indexOf("_-_Site_Ownership_Certificate") > // 0 || // files[prop][0].fieldName.indexOf("_-_Decision_Notice") > 0 || // files[prop][0].fieldName.indexOf("_-_Site_Location_Plan") > 0 || // files[prop][0].fieldName.indexOf("_-_Plans_Drawing_Documents") > // 0 || // files[prop][0].fieldName.indexOf( // "_-_Additional_Plans_Drawings_Documents" // ) > 0 || // files[prop][0].fieldName.indexOf("_-_Design_and_Access_Statement") > // 0 || // files[prop][0].fieldName.indexOf( // "_-_NSB_LPA_Additional_Documents" // ) > 0 || // files[prop][0].fieldName.indexOf("_-_LPA_Correspondence") > 0 || // files[prop][0].fieldName.indexOf("_-_LPA_Original_Permission") > // 0 || // files[prop][0].fieldName.indexOf("_-_LPA's_Registration_Letter") > // 0 || // files[prop][0].fieldName.indexOf("_-_Environmental_Statement") > // 0 || // files[prop][0].fieldName.indexOf("_-_Cost_of_Application") > 0 || // files[prop][0].fieldName.indexOf("_-_Other_Relevant_Material") > // 0 || // files[prop][0].fieldName.indexOf( // "_-_S106_Agreement_or_Unilateral_Undertaking" > 0 // ) // ? "846040000" // : files[prop][0].fieldName.indexOf("_IP_") > 0 // ? "846040005" // : files[prop][0].fieldName.indexOf("_Statement_") > 0 // ? "846040002" // : files[prop][0].fieldName.indexOf("_Questionnaire_") > 0 // ? "846040001" // : files[prop][0].fieldName.indexOf("_Comments_") > 0 // ? "846040003" // : files[prop][0].fieldName.indexOf("_Impact_") > 0 // ? "846040001" // : "846040000"; const fieldName = files[prop][0].fieldName; // Define the patterns and corresponding values const patternMapping = [ { pattern: /_Statement_of_Case|_-_Statement_of_Case|_-_Application_Form|_-_Site_Ownership_Certificate|_-_Decision_Notice|_-_Site_Location_Plan|_-_Plans_Drawing_Documents|_-_Additional_Plans_Drawings_Documents|_-_Design_and_Access_Statement|_-_NSB_LPA_Additional_Documents|_-_LPA_Correspondence|_-_LPA_Original_Permission|_-_LPA's_Registration_Letter|_-_Environmental_Statement|_-_Cost_of_Application|_-_Other_Relevant_Material|_-_S106_Agreement_or_Unilateral_Undertaking/, value: "846040000", }, { pattern: /_IP_/, value: "846040005" }, { pattern: /_Statement_/, value: "846040002" }, { pattern: /_Questionnaire_/, value: "846040001" }, { pattern: /_Comments_/, value: "846040003" }, { pattern: /_Impact_/, value: "846040001" }, ]; // Find a matching pattern or fallback to default let whichLocation = "846040000"; // Default value for (const { pattern, value } of patternMapping) { if (pattern.test(fieldName)) { whichLocation = value; break; } } // return whichLocation; const tags = { containerid: containerName, caseID: foldername.split("/")[0], documentType: files[prop][0].fieldName.split(".")[1], blobType: "RepresentationFile", ishareLocation: whichLocation, documentLocationId: whichLocation, }; const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); console.log( `Uploaded block blob ${files[prop][0].fieldName} successfully`, uploadBlobResponse.requestId ); blobResponseArr.push({ file: files[prop][0].fieldName }); // return uploadBlobResponse; } return blobResponseArr; }; export const uploadRepFiles = async ( formContent, containerName, foldername ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const files = formContent; console.log(...formContent); console.log("rep files...", files, Object.keys(files).length, foldername); const blobName = foldername + "/" + formContent.split("tmp/")[1]; const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadFile(formContent); const tags = { containerid: containerName, caseID: formContent.split("tmp/")[0], documentType: "Rep docs", }; console.log(tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); console.log( `Uploaded block blob ${files} successfully`, uploadBlobResponse.requestId ); }; export const uploadPDFRepFiles = async ( formContent, containerName, foldername, repType ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const files = formContent; console.log(formContent, containerName, foldername); console.log( "rep pdf files...", files, Object.keys(files).length, foldername ); //console.log(`files[${prop}] = ${files[prop][0].size}`); console.log(foldername + "/files/" + formContent.split("tmp/")[1]); const blobName = foldername + "/" + formContent.split("tmp/")[1]; console.log(blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadFile(formContent); const tags = { containerid: containerName, caseID: foldername.split("/")[0], documentType: repType, blobType: "RepresentationPDF", }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); console.log( `Uploaded block blob ${files} successfully`, uploadBlobResponse.requestId ); }; export const uploadPDFAppealFiles = async ( formContent, containerName, foldername, repType ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const files = formContent; //console.log( // "///////////////////////////// \n", // formContent, // containerName, // foldername, // "\n///////////////////////////// \n" // ); //console.log( // "///////////////////////////// \n", // "appeal pdf files...", // files, // Object.keys(files).length, // foldername, // "\n///////////////////////////// \n" // ); //console.log(`files[${prop}] = ${files[prop][0].size}`); console.log(foldername + "/files/" + formContent.split("tmp/")[1]); const blobName = foldername + "/files/" + formContent.split("tmp/")[1]; console.log(blobName); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadFile(formContent); const tags = { containerid: containerName, caseID: foldername.split("/")[0], blobType: "AppealPDF", }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); console.log( `Uploaded block blob ${files} successfully`, uploadBlobResponse.requestId ); }; export const uploadPDFCaseFiles = async ( formContent, containerName, foldername ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const files = formContent; console.log(formContent, containerName, foldername); console.log( "case pdf files...", files, Object.keys(files).length, foldername ); const blobName = foldername + "/files/" + formContent.split("tmp/")[1]; const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadBlobResponse = await blockBlobClient.uploadFile(formContent); const tags = { containerid: containerName, caseID: foldername.split("/")[0], blobType: "AppealPDF", }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); console.log( `Uploaded block blob ${files} successfully`, uploadBlobResponse.requestId ); }; export const downloadFile = async (containerName, blobName) => { //console.log(containerName, blobName); const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const blobClient = containerClient.getBlobClient(blobName); const downloadedBlob = await blobClient.download(0); const downloaded = await streamToBuffer(downloadedBlob.readableStreamBody); return downloaded; }; export const downloadProgressFile = async ( containerName, blobName, casefolderID ) => { console.log(containerName, blobName); const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const blobClient = containerClient.getBlobClient(blobName); const downloadedBlob = await blobClient.download(); const downloaded = await streamToBuffer(downloadedBlob.readableStreamBody); //console.log("Downloaded blob content:", downloaded.toString()); return JSON.parse(downloaded.toString()); }; export const downloadCaseFile = async ( containerName, blobName, casefolderID ) => { console.log(containerName, blobName); const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const blobClient = containerClient.getBlobClient(blobName); const downloadedBlob = await blobClient.download(); const downloaded = await streamToBuffer(downloadedBlob.readableStreamBody); //console.log("Downloaded blob content:", downloaded.toString()); return JSON.parse(downloaded.toString()); }; export const downloadAllProgressFiles = async ( containerName, progressBlobObj ) => { //console.log( // "/////////////////////////\n downloading files: " + // JSON.stringify(progressBlobObj) + // "\n/////////////////////////\n" // ); const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); let blobClient = {}; let downloadedBlob = {}; let downloaded = ""; let caseBlob = {}; let caseDownloaded = ""; let blobCount = 0; for (const prop in progressBlobObj) { //console.log(`Progress - ${prop}: ${progressBlobObj[prop].path}`); blobClient = containerClient.getBlobClient(progressBlobObj[prop].path); downloadedBlob = await blobClient.download(0); downloaded = downloaded + (await streamToBuffer(downloadedBlob.readableStreamBody)) + ","; blobCount++; } downloaded = downloaded.substring(0, downloaded.length - 1); for (const prop in progressBlobObj) { //console.log(`Cases - ${prop}: ${progressBlobObj[prop].caseObj}`); blobClient = containerClient.getBlobClient( progressBlobObj[prop].caseObj ); try { //console.log("===========\nGetting : ", blobClient._name); caseBlob = await blobClient.download(0); caseDownloaded = caseDownloaded + (await streamToBuffer(caseBlob.readableStreamBody)) + ","; } catch { console.log("===========\nFAILED : ", blobClient._name); } } caseDownloaded = caseDownloaded.substring(0, caseDownloaded.length - 1); return JSON.parse( '{ "@odata.count": ' + blobCount + ',"value": [' + downloaded + '],"case":[' + caseDownloaded + "]}" ); }; export const downloadRepFile = async ( containerName, blobName, casefolderID ) => { console.log(containerName, blobName); const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const blobClient = containerClient.getBlobClient(blobName); const downloadedBlob = await blobClient.download(); const downloaded = await streamToBuffer(downloadedBlob.readableStreamBody); console.log("Downloaded blob content:", downloaded.toString()); return JSON.parse(downloaded.toString()); }; export const downloadAllRepsFiles = async (containerName, repsBlobObj) => { //console.log( // "/////////////////////////\n downloading files: " + // JSON.stringify(repsBlobObj) + // "\n/////////////////////////\n" // ); const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); let blobClient = {}; let downloadedBlob = {}; let repArr = []; let downloaded = []; let caseBlob = {}; let caseDownloaded = ""; let blobCount = 0; Array.isArray(repsBlobObj) && !repsBlobObj.length && console.log("No partial reps for this container:" + containerName); for (const prop in repsBlobObj) { //console.log(`Progress - ${prop}: ${repsBlobObj[prop].path}`); blobClient = containerClient.getBlobClient(repsBlobObj[prop].path); downloadedBlob = await blobClient.download(0); let repBlobObj = await streamToBuffer( downloadedBlob.readableStreamBody ); repBlobObj = JSON.parse(repBlobObj); repBlobObj["casereference"] = repsBlobObj[prop].name.split("/")[1]; // repBlobObj = JSON.stringify(repBlobObj); downloaded.push(repBlobObj); blobCount++; } //downloaded = downloaded.substring(0, downloaded.length - 1); //console.log( // "/////////////////////////\n downloaded : " + // JSON.stringify( // '{ "@odata.count": ' + // blobCount + // ',"value": ' + // downloaded + // "}" // ) + // "\n/////////////////////////\n" // ); return { "@odata.count": blobCount, "value": downloaded, }; }; const streamToBuffer = async (readableStream) => { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); }; export const getCaseBlob = async ( containerName, caseReference, formContent ) => { const content = JSON.stringify(formContent); const blobName = caseReference + "/" + caseReference + "_case.json"; const containerBlobToken = await createBlobSas(containerName, blobName); const blobSasUrl = `${STORAGE_PATH}/${containerName}/${blobName}?${containerBlobToken}`; const blockBlobClient = new BlockBlobClient(blobSasUrl); const uploadBlobResponse = await blockBlobClient.upload( content, Buffer.byteLength(content) ); const tags = { containerid: containerName, caseID: caseReference, blobType: "Case", }; //console.log("the tags:", tags); const withTags = await blockBlobClient.setTags(tags); const withMeta = await blockBlobClient.setMetadata(tags); return blobName; }; export const getTempCaseBlob = async (containerName, caseReference) => { const blobName = caseReference + "/" + caseReference + "_case.json"; const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); const blobClient = containerClient.getBlobClient(blobName); const downloadedBlob = await blobClient.download(0); const downloaded = await streamToBuffer(downloadedBlob.readableStreamBody); return JSON.parse(downloaded.toString()); }; export const getProgressBlobs = async (containerName, caseReference) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); let blobCount = 0; for await (const blob of containerClient.listBlobsFlat({ prefix: caseReference, })) { blobCount++; } //console.log( // "this is the casefolder and blobcount:", // caseReference, // blobCount // ); let blobObj = []; for await (const blob of containerClient.listBlobsFlat({ prefix: caseReference + "/" + caseReference + "_appeal.json", })) { console.log("getProgressBlobs in here", blob.name.split("/")); blobObj.push({ "name": blob.name.split("/")[1], "path": blob.name, "versionId": blob.versionId, "caseObj": caseReference + "/" + caseReference + "_case.json", "isCurrentVersion": blob.isCurrentVersion, "contentLength": blob.properties.contentLength, "contentType": blob.contentType, "lastModified": blob.properties.lastModified, "hashedfilepath": hashAPIPath( "/api/file/downloadblob?container=" + containerName + "&casefolderID=" + encodeURIComponent(caseReference) + "&blobname=" + encodeURIComponent(blob.name.split("/")[1]) ), "hasheddeletepath": hashAPIPath( "/api/file/deleteblob?container=" + containerName + "&casefolderID=" + encodeURIComponent(caseReference) + "&blobname=" + encodeURIComponent(blob.name.split("/")[1]) ), "hashgetblobs": hashAPIPath( "/api/file/getbloblist?container=" + containerName + "&casefolderID=" + caseReference ), }); } blobObj = _.sortBy(blobObj, [ function (o) { return o.lastModified; }, ]).reverse()[0]; return blobObj; }; export const getAllProgressBlobs = async (containerName) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); let blobCount = 0; let blobObj = []; for await (const blob of containerClient.findBlobsByTags( "blobType='Appeal'" )) { blob.name.split("/")[1].indexOf("_appeal.json") > 0 && blob.name.split("/")[1].indexOf("undefined") < 0 && blobObj.push({ "name": blob.name.split("/")[1], "path": blob.name, //"versionId": blob.versionId, "caseObj": blob.name.split("/")[0] + "/" + blob.name.split("/")[0] + "_case.json", // "isCurrentVersion": blob.isCurrentVersion, // "contentLength": blob.properties.contentLength, // "contentType": blob.contentType, // "lastModified": blob.properties.lastModified, }); } // console.log( // "\n////////////////////////////\n", // sasUrl, // "blobObj:", // blobObj, // "\n////////////////////////////" // ); function removeDuplicates(arr) { const unique = arr.filter( (value, index, self) => index === self.findIndex( (t) => t.name === value.name // Change 'id' to the property you're checking for uniqueness ) ); return unique; } const result = removeDuplicates(blobObj); return result; }; export const getRepsBlobs = async (containerName) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); let blobCount = 0; for await (const blob of containerClient.findBlobsByTags( "blobType='Representation'" )) { //console.log(`-----Get Rep Blob ${blobCount++}: ${blob.name}`); blobCount++; } //console.log( // "this is the casefolder and blobcount:", // //caseReference, // blobCount // ); const listOptions = { includeMetadata: true, includeSnapshots: false, includeTags: true, includeVersions: true, }; let blobObj = []; for await (const blob of containerClient.findBlobsByTags( "blobType='Representation'", listOptions )) { const blobClient = containerClient.getBlobClient(blob.name); //console.log("getreps blob:", blob); blob.name.split("/")[2].indexOf("_rep.json") > 0 && blob.name.split("/")[2].indexOf("undefined") < 0 && blobObj.push({ "name": blob.name.split("/")[2], "path": blob.name, "size": blobClient.getProperties().contentLength, }); } //console.log("blobObjwwwww:", blobObj); return blobObj; }; export const getRepsFilesBlobs = async ( containerName, casefolderID, filenamePrefix ) => { const containerToken = await createContainerSas(containerName); const sasUrl = `${STORAGE_PATH}/${containerName}?${containerToken}`; const containerClient = new ContainerClient(sasUrl); //conLog"getBlobs " + sasUrl); //console.log(containerClient.listBlobsFlat()); const blobObj = []; for await (const blob of containerClient.listBlobsFlat({ prefix: casefolderID + "/" + filenamePrefix + "/files/", })) { let blobDocumentType = blob.name .split("/")[2] .slice(0, blob.name.split("/")[2].indexOf("_")); console.log(blob.name); blobObj.push({ "name": blob.name.split("/")[3], "path": blob.name, "documentType": getDocumentTypeFromFilename( blob.name.split("/")[3] ), "versionId": blob.versionId, "isCurrentVersion": blob.isCurrentVersion, "contentLength": blob.properties.contentLength, "filenameprefix": filenamePrefix, "filepath": "/api/file/downloadblob?container=" + containerName + "&casefolderID=" + encodeURIComponent( blob.name.split("/")[0] + "/" + blob.name.split("/")[1] ) + "&blobname=" + encodeURIComponent(blob.name.split("/")[3]), "hashedfilepath": hashAPIPath( "/api/file/downloadblob?container=" + containerName + "&casefolderID=" + encodeURIComponent( blob.name.split("/")[0] + "/" + blob.name.split("/")[1] ) + "&blobname=" + encodeURIComponent(blob.name.split("/")[3]) ), "deletepath": "/api/file/deleteblob?container=" + containerName + "&casefolderID=" + blob.name.split("/")[0] + "/" + blob.name.split("/")[1] + "&blobname=" + blob.name.split("/")[3], "hasheddeletepath": hashAPIPath( "/api/file/deleteblob?container=" + containerName + "&casefolderID=" + encodeURIComponent( blob.name.split("/")[0] + "/" + blob.name.split("/")[1] ) + "&blobname=" + encodeURIComponent(blob.name.split("/")[3]) ), "hashgetblobs": hashAPIPath( "/api/file/getbloblist?container=" + containerName + "&casefolderID=" + blob.name.split("/")[0] + "/" + blob.name.split("/")[1] ), }); } console.log("blobObj:", blobObj); return blobObj; }; export const createQueueSas = async (queueName) => { // Get environment variables const account = process.env.AZURE_STORAGE_ACCOUNT_NAME; const accountKey = process.env.AZURE_STORAGE_ACCOUNT_KEY; const sharedKeyCredential = new StorageSharedKeyCredential( account, accountKey ); // Best practice: create time limits const TEN_MINUTES = 10 * 60 * 1000; const NOW = new Date(); // Best practice: set start time a little before current time to // make sure any clock issues are avoided const TEN_MINUTES_AFTER_NOW = new Date(NOW.valueOf() + TEN_MINUTES); var resource_types = new AccountSASResourceTypes(); resource_types.container = true; resource_types.object = true; resource_types.service = true; var permission = new QueueSASPermissions(); permission.read = true; permission.write = true; permission.delete = true; permission.list = true; permission.add = true; permission.update = true; permission.process = true; var sas_signature_val = { expiresOn: TEN_MINUTES_AFTER_NOW, permissions: permission, queueName: queueName, }; const sasToken = generateQueueSASQueryParameters( sas_signature_val, sharedKeyCredential ); //console.log("sass token:", sasToken); return sasToken.toString(); }; export const createCaseCompleteMessage = async ( containerName, caseReference ) => { const whichQueue = "pedw-submitted-applications"; const queueToken = await createQueueSas(whichQueue); const sasUrl = `${QUEUE_PATH}?${queueToken}`; const queueServiceClient = new QueueServiceClient(sasUrl); const endpointURL = process.env.AZURE_PEDW_STORAGE_ENDPOINT; const message = { containerName: containerName, appealpath: endpointURL + "/" + containerName + "/" + caseReference + "/" + caseReference + "_appeal.json", casepath: endpointURL + "/" + containerName + "/" + caseReference + "/" + caseReference + "_case.json", filespath: endpointURL + "/" + containerName + "/" + caseReference + "/files", // uploadUrl: fileRecord.uploadUrl, // filename: fileRecord.file.name, // fileSize: fileRecord.file.size, }; console.log( "/////////////////////\n", JSON.stringify( `${JSON.stringify( message )}` ), "/////////////////////\n" ); const sendMessageResponse = await queueServiceClient .getQueueClient(whichQueue) .sendMessage( JSON.stringify( `${JSON.stringify( message )}` ) ); return sendMessageResponse; }; export const createRepCompleteMessage = async ( containerName, caseReference, filename ) => { const whichQueue = "pedw-submitted-representations"; const queueToken = await createQueueSas(whichQueue); const sasUrl = `${QUEUE_PATH}?${queueToken}`; const queueServiceClient = new QueueServiceClient(sasUrl); const endpointURL = process.env.AZURE_PEDW_STORAGE_ENDPOINT; const message = { containerName: containerName, caseref: caseReference, reppath: endpointURL + "/" + containerName + "/" + caseReference + "/" + filename, filespath: endpointURL + "/" + containerName + "/" + caseReference + "/" + filename + "/files", }; console.log("======================", message, "====================="); const sendMessageResponse = await queueServiceClient .getQueueClient(whichQueue) .sendMessage( JSON.stringify( `${JSON.stringify( message )}` ) ); return sendMessageResponse; }; export const listContainers = async ( blobServiceClient, containerNamePrefix ) => { const options = { includeDeleted: false, includeMetadata: true, includeSystem: true, prefix: containerNamePrefix, }; console.log( "\n\n\n=================================================\n", "Showing storage account contents:\n\n" ); for await (const containerItem of blobServiceClient.listContainers( options )) { // ContainerItem console.log(`For-await list: ${containerItem.name}`); // ContainerClient const containerClient = blobServiceClient.getContainerClient( containerItem.name ); await listBlobHierarchical(containerClient).catch((error) => consoleLogger(error) ); // ... do something with container } console.log("\n=================================================\n\n\n\n"); }; export const listBlobHierarchical = async ( containerClient, virtualHierarchyDelimiter = "/" ) => { // page size - artificially low as example const maxPageSize = 2; // some options for filtering list const listOptions = { includeCopy: false, // include metadata from previous copies includeDeleted: false, // include deleted blobs includeDeletedWithVersions: false, // include deleted blobs with versions includeLegalHold: false, // include legal hold includeMetadata: true, // include custom metadata includeSnapshots: false, // include snapshots includeTags: true, // include indexable tags includeUncommitedBlobs: false, // include uncommitted blobs includeVersions: false, // include all blob version prefix: "", // filter by blob name prefix }; let i = 1; console.log(`Folder ${virtualHierarchyDelimiter}`); for await (const response of containerClient .listBlobsByHierarchy(virtualHierarchyDelimiter, listOptions) .byPage({ maxPageSize })) { console.log(` Blob number ${i++}`); const segment = response.segment; if (segment.blobPrefixes) { // Do something with each virtual folder for await (const prefix of segment.blobPrefixes) { // build new virtualHierarchyDelimiter from current and next await listBlobHierarchical( containerClient, `${virtualHierarchyDelimiter}${prefix.name}` ); } } for (const blob of response.segment.blobItems) { // Do something with each blob console.log(`\tBlobItem: name - ${blob.name}`); } } }; function formatTimeSince(date) { if (!date) return "Unknown date"; const now = new Date(); const diffMs = now - date; const diffDays = Math.floor(diffMs / (1000 * 60 * 60 * 24)); if (diffDays < 7) { return `${diffDays} day${diffDays !== 1 ? "s" : ""} ago`; } else { const diffWeeks = Math.floor(diffDays / 7); return `${diffWeeks} week${diffWeeks !== 1 ? "s" : ""} ago`; } } function formatDateDisplay(date) { return date ? `${date.toLocaleString("en-GB")} (${formatTimeSince(date)})` : "Unknown"; } export async function listBlobHierarchicalForUser(containerClient) { const blobNames = []; for await (const blob of containerClient.listBlobsFlat()) { const lastModifiedDate = blob.properties?.lastModified || null; const createdOnDate = blob.properties?.createdOn || null; let displayDate = ""; if (createdOnDate && lastModifiedDate) { if ( createdOnDate.toDateString() === lastModifiedDate.toDateString() ) { // Same day — show just one displayDate = `Date: ${formatDateDisplay(lastModifiedDate)}`; } else { // Different days — show both displayDate = `Created: ${formatDateDisplay( createdOnDate )}, Modified: ${formatDateDisplay(lastModifiedDate)}`; } } else if (lastModifiedDate) { displayDate = `Modified: ${formatDateDisplay(lastModifiedDate)}`; } else if (createdOnDate) { displayDate = `Created: ${formatDateDisplay(createdOnDate)}`; } else { displayDate = "Unknown date"; } blobNames.push({ name: blob.name, size: blob.properties.contentLength || 0, lastModified: lastModifiedDate ? lastModifiedDate.toISOString() : null, createdOn: createdOnDate ? createdOnDate.toISOString() : null, displayDate, }); } return blobNames; } export async function listContainersForUser( blobServiceClient, containerNamePrefix, emailAddress ) { const options = { includeDeleted: false, includeMetadata: true, includeSystem: true, prefix: containerNamePrefix, }; const results = []; for await (const containerItem of blobServiceClient.listContainers( options )) { const containerClient = blobServiceClient.getContainerClient( containerItem.name ); const blobNames = await listBlobHierarchicalForUser( containerClient ).catch(console.error); if (blobNames && blobNames.length > 0) { results.push({ container: containerItem.name, blobs: blobNames, }); } } return results; // array of non-empty containers }