diff --git a/.gitignore b/.gitignore index 7485fbf5..ad55dc2c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ public/swagger.json certificates/cert-key.pem certificates/cert.pem gitclean.sh +pages/admin/logcheck.js +pages/admin/logchecker.js +pages/baracuda.html diff --git a/actions/index.js b/actions/index.js index e995a893..fd04e4a8 100644 --- a/actions/index.js +++ b/actions/index.js @@ -2144,8 +2144,25 @@ export const getNewDocumentsPaged = async ( fieldSort, showNumberOfRecords, documentType, + documentOrigin, selectedWeeks ) => { + console.log( + "/api/admin/getlatestdocuments_api?pageNumber=" + + pageNumber + + "&orderby=" + + orderBy + + "&fieldSort=" + + fieldSort + + "&showNumberOfRecords=" + + showNumberOfRecords + + "&documentType=" + + documentType + + "&numberWeeks=" + + selectedWeeks + + "&documentOrigin=" + + documentOrigin + ); return axios .get( "/api/admin/getlatestdocuments_api?pageNumber=" + @@ -2159,7 +2176,9 @@ export const getNewDocumentsPaged = async ( "&documentType=" + documentType + "&numberWeeks=" + - selectedWeeks + selectedWeeks + + "&documentOrigin=" + + documentOrigin ) .then((res) => { return res.data; diff --git a/components/admin/tabs/accounts.js b/components/admin/tabs/accounts.js index 97498248..8d802860 100644 --- a/components/admin/tabs/accounts.js +++ b/components/admin/tabs/accounts.js @@ -26,6 +26,8 @@ export default function AccountsTab({ userData }) { > {user.email} +
+ {user.id}
Last accessed: {user.created} diff --git a/components/admin/tabs/documents.js b/components/admin/tabs/documents.js index 23818da2..94bc359a 100644 --- a/components/admin/tabs/documents.js +++ b/components/admin/tabs/documents.js @@ -7,6 +7,9 @@ import { connect } from "react-redux"; import transLookup from "../../../data/lookuptranslations.json"; import { formatDates } from "../../utils"; import PaginationControl from "../../../components/case/pagination"; +import DocumentLink from "../../utils/downloads"; +import { useDownloadQueue } from "../../utils/downloadmanager"; + import { sendGAEvent } from "@next/third-parties/google"; import { @@ -170,6 +173,21 @@ const getDocumentTypes = [ }, ]; +const getOrigin = [ + { + "Value": 846040000, + "Label": "MDU", + }, + { + "Value": 846040001, + "Label": "Portal", + }, + { + "Value": 846040002, + "Label": "Manual", + }, +]; + const DocumentsTab = (props) => { let { t, lang } = useTranslation(); const firstRender = useRef(false); @@ -189,12 +207,15 @@ const DocumentsTab = (props) => { const { locale } = router; var documentDetailsArr = documentDetailsObj || []; - + const { startDownload, getStatus } = useDownloadQueue(3); const [selectedOption, setSelectedOption] = useState(10); const [selectedWeeks, setSelectedWeeks] = useState(1); const [documentTypes, setDocumentTypes] = useState(getDocumentTypes); + const [documentOrigin, setDocumentOrigin] = useState(getOrigin); const [selectedDocumentType, setSelectedDocumentType] = useState("all"); + const [selectedDocumentOrigin, setSelectedDocumentOrigin] = useState("all"); const [selectAll, setSelectAll] = useState(false); + const [selectOriginAll, setSelectOriginAll] = useState(false); const [checkedItems, setCheckedItems] = useState( documentTypes.reduce((acc, item) => { @@ -203,6 +224,13 @@ const DocumentsTab = (props) => { }, {}) ); + const [checkedOriginItems, setCheckedOriginItems] = useState( + documentOrigin.reduce((acc, item) => { + acc[item.pinswg_origin] = false; + return acc; + }, {}) + ); + const handleCheckboxChange = (e) => { const { name, checked } = e.target; @@ -213,6 +241,16 @@ const DocumentsTab = (props) => { setSelectAll(false); }; + const handleCheckboxOriginChange = (e) => { + const { name, checked } = e.target; + + setCheckedOriginItems((prev) => ({ + ...prev, + [name]: checked, + })); + setSelectOriginAll(false); + }; + const handleSelectAllChange = (e) => { const { checked } = e.target; setSelectAll(checked); // Set the 'Select All' state @@ -230,16 +268,42 @@ const DocumentsTab = (props) => { } }; + const handleSelectOriginAllChange = (e) => { + const { checked } = e.target; + setSelectOriginAll(checked); // Set the 'Select All' state + if (checked) { + // If 'Select All' is checked, check all the other checkboxes + + setCheckedOriginItems( + documentOrigin.reduce((acc, item) => { + acc[item.pinswg_origin] = true; + return acc; + }, {}) + ); + } else { + clearCheckboxes(); + } + }; + const clearCheckboxes = () => { setSelectAll(false); + setSelectOriginAll(false); setCheckedItems( documentTypes.reduce((acc, item) => { acc[item.pinswg_isharedocumentlocations] = false; return acc; }, {}), + setSelectedDocumentType("all"), setCurrentPage(1) ); + setCheckedOriginItems( + documentOrigin.reduce((acc, item) => { + acc["origin_" + item.pinswg_origin] = false; + return acc; + }, {}), + setSelectedDocumentOrigin("all") + ); }; let ShowDocLinks = false; let checkShowDocLinks = docsOffline != false ? true : false; @@ -314,6 +378,9 @@ const DocumentsTab = (props) => { {t("search:clear-filter-button-label")} +

+ Document Type +

{" "} <> @@ -335,17 +402,6 @@ const DocumentsTab = (props) => { {router.locale == "cy" ? "Pawb" : "All"}{" "} - - ( - {documentTypes.reduce( - (accumulator, currentItem) => { - return ( - accumulator + - currentItem.count - ); - }, - 0 - )} - )
@@ -389,17 +445,105 @@ const DocumentsTab = (props) => { ); })} {" "} +

+ Document Origin +

+
+ {" "} + <> +
+ + +
+ + {documentOrigin.map((item, key) => { + return ( +
+ + +
+ ); + })} +
{" "}
+// // ) : ( +// // <> +// // +// // Document name: +// // +// // {detailsObj.pinswg_name || ""} +// // +// // )} + +// // {downloadStatus && ( +// //

+// // {downloadStatus} +// // {/* {progress !== null && ` (${progress}%)`} */} +// //

+// // )} +// // +// // ); +// // } +// import { useState } from "react"; + +// export default function DocumentLink({ +// id, +// detailsObj, +// caseReference, +// startDownload, +// getStatus, +// }) { +// const [downloadStatus, setDownloadStatus] = useState(""); +// const [progress, setProgress] = useState(null); + +// const downloadFile = async () => { +// setDownloadStatus("Downloading"); +// setProgress(0); + +// try { +// const res = await fetch(detailsObj.pinswg_hashlink); +// if (!res.ok) throw new Error("Download failed"); + +// const contentDisposition = res.headers.get("content-disposition"); +// const filename = contentDisposition +// ? contentDisposition.split("filename=")[1] +// : detailsObj.pinswg_name || "document"; + +// const contentLength = res.headers.get("content-length"); +// const totalBytes = contentLength +// ? parseInt(contentLength, 10) +// : null; + +// const reader = res.body.getReader(); +// let receivedBytes = 0; +// const chunks = []; + +// while (true) { +// const { done, value } = await reader.read(); +// if (done) break; + +// chunks.push(value); +// receivedBytes += value.length; + +// if (totalBytes) { +// const percent = Math.round( +// (receivedBytes / totalBytes) * 100 +// ); +// setProgress(percent); +// } +// } + +// const blob = new Blob(chunks); +// const url = window.URL.createObjectURL(blob); + +// const link = document.createElement("a"); +// link.href = url; +// link.download = filename; +// document.body.appendChild(link); +// link.click(); +// document.body.removeChild(link); + +// setDownloadStatus("Download complete!"); +// setProgress(100); + +// window.gtag?.("event", "DownloadedFile", { +// caseReference, +// filename, +// }); +// } catch (err) { +// console.error(err); +// setDownloadStatus("Download failed"); +// setProgress(null); +// } +// }; + +// const handleClick = () => { +// if (getStatus(id) !== "idle") return; // don't double enqueue +// startDownload(id, downloadFile); +// }; + +// const status = getStatus(id); + +// return ( +// <> +// {detailsObj.pinswg_publishtoweb ? ( +// +// ) : ( +// <> +// +// Document name: +// +// {detailsObj.pinswg_name || ""} +// +// )} + +// {status !== "idle" && ( +//

+// {status === "downloading" && progress !== null +// ? `Downloading (${progress}%)` +// : status === "queued" +// ? "Queued" +// : status === "done" +// ? "Download complete!" +// : status === "failed" +// ? "Download failed" +// : ""} +//

+// )} +// +// ); +// } + +import { useState } from "react"; +import useTranslation from "next-translate/useTranslation"; + +export default function DocumentLink({ + id, + detailsObj, + caseReference, + startDownload, + getStatus, +}) { + const [progress, setProgress] = useState(null); + let { t } = useTranslation(); + const downloadFile = async () => { + setProgress(0); + + const res = await fetch(detailsObj.pinswg_hashlink); + if (!res.ok) throw new Error("Download failed"); + + const contentDisposition = res.headers.get("content-disposition"); + const filename = contentDisposition + ? contentDisposition.split("filename=")[1] + : detailsObj.pinswg_name || "document"; + + const contentLength = res.headers.get("content-length"); + const totalBytes = contentLength ? parseInt(contentLength, 10) : null; + + const reader = res.body.getReader(); + let receivedBytes = 0; + const chunks = []; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + chunks.push(value); + receivedBytes += value.length; + + if (totalBytes) { + const percent = Math.round((receivedBytes / totalBytes) * 100); + setProgress(percent); + } + } + + const blob = new Blob(chunks); + const url = window.URL.createObjectURL(blob); + + const link = document.createElement("a"); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + window.URL.revokeObjectURL(url); + + setProgress(100); + window.gtag?.("event", "DownloadedFile", { caseReference, filename }); + }; + + const handleClick = () => { + if (getStatus(id) !== "idle") return; // avoid double enqueue + startDownload(id, downloadFile); + }; + + const status = getStatus(id); + + return ( +
+ {detailsObj.pinswg_publishtoweb ? ( + + ) : ( + + {detailsObj.pinswg_name || ""} + + )} + + {status !== "idle" && ( +

+ {status === "downloading" && progress !== null + ? t("case:downloading-label") + : status === "queued" + ? t("case:download-queued-label") + : status === "done" + ? t("case:download-complete-label") + : status === "failed" + ? t("case:download-failed-label") + : ""} +

+ )} +
+ ); +} diff --git a/locales/cy/case.json b/locales/cy/case.json index a0490f05..da629c24 100644 --- a/locales/cy/case.json +++ b/locales/cy/case.json @@ -100,5 +100,9 @@ "do-you-want-to-delete-case-disclaimer-label": "Ar ôl dileu, bydd yr holl wybodaeth a gyflwynwyd gennych yn cael ei cholli.", "representation-date-passed-label": "Mae'r cyfnod cyflwyno sylwadau ar gyfer yr apêl hon rhwng {{startDate}} a {{endDate}}", "representation-date-passed-additional-label": "Gallwch wneud sylw drwy e-bostio,", - "date-raised": "Dyddiad codi" + "date-raised": "Dyddiad codi", + "download-complete-label": "Lawrlwytho wedi'i gwblhau!", + "downloading-label": "Wrthi'n llwytho i lawr", + "download-failed-label": "Methodd y lawrlwythiad", + "download-queued-label": "Wedi'i giwio" } \ No newline at end of file diff --git a/locales/en/case.json b/locales/en/case.json index fac349c1..25e02d0c 100644 --- a/locales/en/case.json +++ b/locales/en/case.json @@ -100,5 +100,9 @@ "do-you-want-to-delete-case-disclaimer-label": "Once deleted all information you have submited, will be lost.", "representation-date-passed-label": "The representation period for this appeal is between {{startDate}} and {{endDate}}", "representation-date-passed-additional-label": "You can make a representation by emailing,", - "date-raised": "Date raised" + "date-raised": "Date raised", + "download-complete-label": "Download complete!", + "downloading-label": "Downloading", + "download-failed-label": "Download failed", + "download-queued-label": "Queued" } \ No newline at end of file diff --git a/pages/admin/index.js b/pages/admin/index.js index ab21835b..86ce577a 100644 --- a/pages/admin/index.js +++ b/pages/admin/index.js @@ -54,10 +54,10 @@ const StoragePage = (props) => { {/* Tabs */} {/* Panels */} + +
+ +
+
+ +
+
{ >
- -
- -
- -
- -
diff --git a/pages/api/admin/getlatestdocuments_api.js b/pages/api/admin/getlatestdocuments_api.js index c5742dab..36b7117c 100644 --- a/pages/api/admin/getlatestdocuments_api.js +++ b/pages/api/admin/getlatestdocuments_api.js @@ -64,9 +64,11 @@ export default async function ApiProxy(req, res) { var fieldSort = req.query.fieldSort || "desc"; var showNumberOfRecords = req.query.showNumberOfRecords || 10; var documentType = req.query.documentType || "all"; + var documentOrigin = req.query.documentOrigin || "all"; var numberOfWeeks = req.query.numberWeeks || 1; var docTypeQueryString = ""; + var docOriginQueryString = ""; if (documentType != "all") { if (documentType.indexOf(",") >= 0) { @@ -91,6 +93,27 @@ export default async function ApiProxy(req, res) { } } + if (documentOrigin != "all") { + if (documentOrigin.indexOf(",") >= 0) { + const documentOriginArr = documentOrigin.split(","); + + docOriginQueryString = " and ("; + documentOriginArr.forEach(function (item, index) { + console.log(item, index); + + if (item != "all") { + docOriginQueryString += " pinswg_origin eq " + item; + + docOriginQueryString += + index < documentOriginArr.length - 1 ? " or " : ""; + } + }); + docOriginQueryString += " )"; + } else { + docOriginQueryString += " and pinswg_origin eq " + documentOrigin; + } + } + function daysAgoISO(days) { const today = new Date(); const resultDate = new Date(today); @@ -103,6 +126,7 @@ export default async function ApiProxy(req, res) { "pinswg_documents?$select=pinswg_name,createdon,pinswg_publishtoweb,_pinswg_documentids_value,pinswg_isharedocumentlocations,pinswg_isharedocumentreference,pinswg_uploadurl,pinswg_uploadstatus,pinswg_origin&$filter=createdon ge " + daysAgoISO(numberOfWeeks) + docTypeQueryString + + docOriginQueryString + "&$orderby=" + orderby + " " + @@ -115,7 +139,7 @@ export default async function ApiProxy(req, res) { console.log( "\n==========================================\n", - "\nnew docs search ", + "\nnew docs search---- ", "\n\nQuery url: " + queryUrl, "\n\nRelay link: " + WEBAPI_URL + queryUrl + hashAPIPath(queryUrl), "\n==========================================\n" diff --git a/pages/api/documents/download/[id].js b/pages/api/documents/download/[id].js index 044e62b9..cbc8db9c 100644 --- a/pages/api/documents/download/[id].js +++ b/pages/api/documents/download/[id].js @@ -1,110 +1,203 @@ -/** - * @swagger - * /api/documents/download/{id}: - * get: - * tags: - * - Documents - * description: Get document - * parameters: - * - name: id - * in: path - * description: iShare ID - * type: string - * required: true - * default: A41567436 - * - name: hash - * in: query - * description: Hash string - * default: e0a1c9cd2817826a25bca67e60b807eae747cbed769e3ec42cf997541c32be71 - * responses: - * 200: - * description: Success - */ +// /** +// * @swagger +// * /api/documents/download/{id}: +// * get: +// * tags: +// * - Documents +// * description: Get document +// * parameters: +// * - name: id +// * in: path +// * description: iShare ID +// * type: string +// * required: true +// * default: A41567436 +// * - name: hash +// * in: query +// * description: Hash string +// * default: e0a1c9cd2817826a25bca67e60b807eae747cbed769e3ec42cf997541c32be71 +// * responses: +// * 200: +// * description: Success +// */ + +// import axios from "axios"; +// import CryptoJS from "crypto-js"; +// import { getToken, consoleLogger } from "../../../../actions"; + +// const WORDKEY = process.env.HASHKEY; +// const accessTokenEndpoint = process.env.ACCESS_TOKEN_ENDPOINT; +// const tenantId = process.env.TENANT; + +// const WEBAPI_URL = +// process.env.RELAY_ROOT || +// "https://dev-pedw-ns.servicebus.windows.net/dev-pedw-hc/"; + +// const hashAPIPath = (queryPath) => { +// var hashlink = CryptoJS.HmacSHA256( +// queryPath, +// CryptoJS.enc.Hex.parse(WORDKEY) +// ); +// hashlink = hashlink.toString(CryptoJS.enc.Hex); + +// //return "&hash=" + hashlink; + +// return (queryPath.indexOf("?") > -1 ? "&hash=" : "?hash=") + hashlink; +// }; + +// const azureHeadersPaged = (access_token) => { +// return { +// headers: { +// "OData-MaxVersion": "4.0", +// "OData-Version": "4.0", +// "Accept": "application/json;odata.metadata=none", +// "Prefer": +// 'odata.include-annotations="*",return=representation, odata.maxpagesize=10', +// "Content-Type": "application/json", +// "Authorization": "Bearer " + access_token, +// }, +// }; +// }; + +// export default async function ApiProxy(req, res) { +// var token = await getToken(); + +// var docRef = req.query; + +// var queryUrl = "documents/download/" + docRef.id + "?hash=" + docRef.hash; + +// const configDocument = (access_token) => { +// return { +// headers: { +// "OData-MaxVersion": "4.0", +// "OData-Version": "4.0", +// "Accept": "application/json", +// "Prefer": 'odata.include-annotations="*",return=representation', +// "Content-Type": "application/json", +// "Authorization": "Bearer " + access_token, +// }, +// responseType: "arraybuffer", +// }; +// }; + +// return axios +// .get( +// WEBAPI_URL + queryUrl, // + hashAPIPath(queryUrl), +// configDocument(token.access_token) +// ) +// .then((response) => { +// const bytes = response.data.byteLength; +// console.log(bytes); + +// res.setHeader( +// "content-disposition", +// "attachment; filename=" + +// response.headers["content-disposition"].split( +// "filename=" +// )[1] +// ); +// return res.status(200).send(response.data); +// }) +// .then(() => { +// console.log(docRef.id + "has downloaded"); + +// return "downloadComplete"; +// }) +// .catch((error) => { +// consoleLogger(error); +// res.redirect("/filenotavailable"); +// }); +// } import axios from "axios"; -import CryptoJS from "crypto-js"; import { getToken, consoleLogger } from "../../../../actions"; -const WORDKEY = process.env.HASHKEY; -const accessTokenEndpoint = process.env.ACCESS_TOKEN_ENDPOINT; -const tenantId = process.env.TENANT; - const WEBAPI_URL = process.env.RELAY_ROOT || "https://dev-pedw-ns.servicebus.windows.net/dev-pedw-hc/"; -const hashAPIPath = (queryPath) => { - var hashlink = CryptoJS.HmacSHA256( - queryPath, - CryptoJS.enc.Hex.parse(WORDKEY) - ); - hashlink = hashlink.toString(CryptoJS.enc.Hex); - - //return "&hash=" + hashlink; - - return (queryPath.indexOf("?") > -1 ? "&hash=" : "?hash=") + hashlink; -}; - -const azureHeadersPaged = (access_token) => { - return { - headers: { - "OData-MaxVersion": "4.0", - "OData-Version": "4.0", - "Accept": "application/json;odata.metadata=none", - "Prefer": - 'odata.include-annotations="*",return=representation, odata.maxpagesize=10', - "Content-Type": "application/json", - "Authorization": "Bearer " + access_token, - }, - }; -}; - -export default async function ApiProxy(req, res) { - var token = await getToken(); - - var docRef = req.query; - - var queryUrl = "documents/download/" + docRef.id + "?hash=" + docRef.hash; - - const configDocument = (access_token) => { - return { - headers: { - "OData-MaxVersion": "4.0", - "OData-Version": "4.0", - "Accept": "application/json", - "Prefer": 'odata.include-annotations="*",return=representation', - "Content-Type": "application/json", - "Authorization": "Bearer " + access_token, - }, - responseType: "arraybuffer", - }; - }; - - return axios - .get( - WEBAPI_URL + queryUrl, // + hashAPIPath(queryUrl), - configDocument(token.access_token) - ) - .then((response) => { - const bytes = response.data.byteLength; - console.log(bytes); - - res.setHeader( - "content-disposition", - "attachment; filename=" + - response.headers["content-disposition"].split( - "filename=" - )[1] - ); - return res.status(200).send(response.data); - }) - .then(() => { - console.log(docRef.id + "has downloaded"); - - return "downloadComplete"; - }) - .catch((error) => { - consoleLogger(error); - res.redirect("/filenotavailable"); - }); +// Retry utility with logging +async function retry(fn, retries = 3, delay = 1000) { + for (let attempt = 1; attempt <= retries; attempt++) { + try { + return { response: await fn(), attempts: attempt }; + } catch (err) { + if (attempt === retries) throw err; + console.log(`Retry ${attempt} failed, retrying in ${delay}ms...`); + await new Promise((res) => setTimeout(res, delay)); + delay *= 2; // exponential backoff + } + } } + +const ApiProxy = async (req, res) => { + const docRef = req.query; + + try { + const token = await getToken(); + const startTime = Date.now(); + + const fetchStream = async () => { + const queryUrl = + "documents/download/" + docRef.id + "?hash=" + docRef.hash; + + return axios.get(WEBAPI_URL + queryUrl, { + headers: { + "OData-MaxVersion": "4.0", + "OData-Version": "4.0", + "Accept": "application/json", + "Prefer": + 'odata.include-annotations="*",return=representation', + "Content-Type": "application/json", + "Authorization": "Bearer " + token.access_token, + }, + responseType: "stream", + }); + }; + + const { response, attempts } = await retry(fetchStream, 3, 1000); + + // Extract filename + const contentDisposition = response.headers["content-disposition"]; + const filename = contentDisposition + ? contentDisposition.split("filename=")[1] + : docRef.id; + + res.setHeader( + "Content-Disposition", + `attachment; filename=${filename}` + ); + res.setHeader("Content-Type", "application/octet-stream"); + + let totalBytes = 0; + + response.data.on("data", (chunk) => { + totalBytes += chunk.length; + }); + + response.data.pipe(res); + + response.data.on("end", () => { + const durationMs = Date.now() - startTime; + console.log( + `[Download Complete] Document: ${filename}, ID: ${docRef.id}, Size: ${totalBytes} bytes, Duration: ${durationMs}ms, Attempts: ${attempts}` + ); + }); + + response.data.on("error", (err) => { + consoleLogger(err); + if (!res.headersSent) res.redirect("/filenotavailable"); + }); + } catch (error) { + consoleLogger(error); + if (!res.headersSent) res.redirect("/filenotavailable"); + } +}; + +export const config = { + api: { + responseLimit: false, + }, +}; + +export default ApiProxy; diff --git a/styles/sass/welshgov/_application.scss b/styles/sass/welshgov/_application.scss index b80401b7..9e2f3b14 100644 --- a/styles/sass/welshgov/_application.scss +++ b/styles/sass/welshgov/_application.scss @@ -1469,6 +1469,22 @@ textarea, top: calc(50% - 10px); } } + + .documentLink { + border: none; + text-align: left; + cursor: pointer; + color: #0360a6; + display: block; + padding-left: 0px; + + &:hover { + background-color: $white; + color: #3b7dc5; + border: none; + padding-left: 0px + } + } } .appealModule { @@ -2932,6 +2948,10 @@ fade { .govuk-label-s { font-size: 14px; } + + &.govuk-checkboxes_short { + height: 180px; + } }