mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
remove some old stuffz
This commit is contained in:
Collin M. Barrett
parent
4abce35bfa
commit
e8ee0528b0
9 changed files with 0 additions and 3813 deletions
File diff suppressed because it is too large
Load diff
107
gs/data.gs
107
gs/data.gs
|
|
@ -1,107 +0,0 @@
|
|||
function refreshData() {
|
||||
var data = ImportJSON("https://raw.githubusercontent.com/collinbarrett/FilterLists/master/data/lists.json", "", "noTruncate,rawHeaders,noInherit");
|
||||
data = fill2dJsonArrayWithNulls(data);
|
||||
sData.clearContents();
|
||||
sData.getRange(1, 1, data.length, data[0].length).setValues(data);
|
||||
concatNamelessComboLists();
|
||||
var colIndexNumDataList = getColIndexNum(sData, "list");
|
||||
sData.getRange(2, 1, data.length, data[0].length).sort(colIndexNumDataList);
|
||||
refreshParsed();
|
||||
refreshPublic();
|
||||
}
|
||||
|
||||
function concatNamelessComboLists() {
|
||||
var numLists = getNumLists(sData);
|
||||
var colIndexNumDataList = getColIndexNum(sData, "list");
|
||||
var colRangeDataList = sData.getRange(2, colIndexNumDataList, numLists, 1).getValues();
|
||||
var colRangeDataRelated = sData.getRange(2, getColIndexNum(sData, "related"), numLists, 1).getValues();
|
||||
for (var i = 0; i < numLists; i++) {
|
||||
if (!colRangeDataList[i][0]) {
|
||||
colRangeDataList[i][0] = colRangeDataRelated[i][0].replace(/,/g, " + ");
|
||||
}
|
||||
}
|
||||
sData.getRange(2, colIndexNumDataList, numLists, 1).setValues(colRangeDataList);
|
||||
}
|
||||
|
||||
function refreshParsed() {
|
||||
var parsedFields = ["tags", "viewUrl", "addUrl", "homeUrl", "forumUrl", "issuesUrl", "email", "donateUrl"];
|
||||
for (var i = 0, parsedFieldsLength = parsedFields.length; i < parsedFieldsLength; i++) {
|
||||
var colIndexParsedUrlType = getColIndex(sParsed, parsedFields[i]);
|
||||
var colRangeParsedUrlType = sParsed.getRange(colIndexParsedUrlType + "2:" + colIndexParsedUrlType);
|
||||
var colIndexDataList = getColIndex(sData, "list");
|
||||
colRangeParsedUrlType.clearContent();
|
||||
var colTopParsedUrlType = sParsed.getRange(colIndexParsedUrlType + "2");
|
||||
switch (parsedFields[i]) {
|
||||
case "tags":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayTags()");
|
||||
break;
|
||||
case "viewUrl":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"viewUrl\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
case "addUrl":
|
||||
var colIndexDataViewUrl = getColIndex(sData, "viewUrl");
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"addUrl\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(Data!" + colIndexDataViewUrl + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(Data!" + colIndexDataViewUrl + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
case "homeUrl":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"homeUrl\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
case "forumUrl":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"forumUrl\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
case "issuesUrl":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"issuesUrl\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
case "email":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"email\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
case "donateUrl":
|
||||
colTopParsedUrlType.setValue("=createHtmlArrayLinkButton(\"donateUrl\",INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(" + colIndexParsedUrlType + "1,Data!A1:1,0),4),\"1\",\"\")),Data!" + colIndexDataList + "2:" + colIndexDataList + ")");
|
||||
break;
|
||||
default:
|
||||
}
|
||||
var colDataParsedUrlType = colRangeParsedUrlType.getDisplayValues();
|
||||
colRangeParsedUrlType.clearContent();
|
||||
colRangeParsedUrlType.setValues(colDataParsedUrlType);
|
||||
}
|
||||
}
|
||||
|
||||
function refreshPublic() {
|
||||
var publicFields = ["list", "tags", "description", "links", "related"];
|
||||
for (var i = 0, publicFieldsLength = publicFields.length; i < publicFieldsLength; i++) {
|
||||
var colIndexPublicPublicField = getColIndex(sPublic, publicFields[i]);
|
||||
var colRangePublicPublicField = sPublic.getRange(colIndexPublicPublicField + "2:" + colIndexPublicPublicField);
|
||||
colRangePublicPublicField.clearContent();
|
||||
var colTopPublicPublicField = sPublic.getRange(colIndexPublicPublicField + "2");
|
||||
switch (publicFields[i]) {
|
||||
case "list":
|
||||
colTopPublicPublicField.setValue("=createHtmlArrayListName(INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"list\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"list\",Data!A1:1,0),4),\"1\",\"\")))");
|
||||
break;
|
||||
case "tags":
|
||||
copyTagsToPublic();
|
||||
break;
|
||||
case "description":
|
||||
if (getColIndex(sData, "descrSourceUrl")) {
|
||||
colTopPublicPublicField.setValue("=ARRAYFORMULA(IF(ISBLANK(INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"\"))),,\"<blockquote cite=\"\"\")&INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"\"))&IF(ISBLANK(INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"\"))),,\"\"\">\")&INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descr\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descr\",Data!A1:1,0),4),\"1\",\"\"))&IF(ISBLANK(INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descrSourceUrl\",Data!A1:1,0),4),\"1\",\"\"))),,\"</blockquote>\"))");
|
||||
} else {
|
||||
colTopPublicPublicField.setValue("=ARRAYFORMULA(INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descr\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"descr\",Data!A1:1,0),4),\"1\",\"\")))");
|
||||
}
|
||||
break;
|
||||
case "links":
|
||||
colTopPublicPublicField.setValue("=ARRAYFORMULA(IF(Parsed!B2:B=\"\",\"\",\"<p>\"&Parsed!B2:B&Parsed!C2:C&Parsed!D2:D&Parsed!E2:E&Parsed!F2:F&Parsed!G2:G&Parsed!H2:H&\"</p>\"))");
|
||||
break;
|
||||
case "related":
|
||||
colTopPublicPublicField.setValue("=createHtmlArrayRelatedLists(INDIRECT(\"Data!\"&SUBSTITUTE(ADDRESS(1,MATCH(\"related\",Data!A1:1,0),4),\"1\",\"2\")&\":\"&SUBSTITUTE(ADDRESS(1,MATCH(\"related\",Data!A1:1,0),4),\"1\",\"\")))");
|
||||
break;
|
||||
default:
|
||||
}
|
||||
var colDataPublicPublicField = colRangePublicPublicField.getDisplayValues();
|
||||
colRangePublicPublicField.clearContent();
|
||||
colRangePublicPublicField.setValues(colDataPublicPublicField);
|
||||
}
|
||||
}
|
||||
|
||||
function copyTagsToPublic() {
|
||||
var colIndexParsedTags = getColIndex(sParsed, "tags");
|
||||
var colIndexPublicTags = getColIndex(sPublic, "tags");
|
||||
copyRange(sParsed, colIndexParsedTags + "2:" + colIndexParsedTags, sPublic, colIndexPublicTags + "2:" + colIndexPublicTags);
|
||||
}
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
var filScoreWeight = [
|
||||
["sumUniqueDailyVisitorsLast30", "1.00"]
|
||||
];
|
||||
|
||||
function refreshFilScores() {
|
||||
copyListToFilScore();
|
||||
copyViewUrlToFilScore();
|
||||
getSumUniqueDailyVisitorsLast30();
|
||||
calculateSumUniqueDailyVisitorsLast30Score();
|
||||
calculateFilScore();
|
||||
copyFilScoreToPublic();
|
||||
}
|
||||
|
||||
function copyListToFilScore() {
|
||||
var colIndexDataList = getColIndex(sData, "list");
|
||||
var colIndexFilScoreList = getColIndex(sFilScore, "list");
|
||||
copyRange(sData, colIndexDataList + "2:" + colIndexDataList, sFilScore, colIndexFilScoreList + "2:" + colIndexFilScoreList);
|
||||
}
|
||||
|
||||
function copyViewUrlToFilScore() {
|
||||
var colIndexDataViewUrl = getColIndex(sData, "viewUrl");
|
||||
var colIndexFilScoreViewUrl = getColIndex(sFilScore, "viewUrl");
|
||||
copyRange(sData, colIndexDataViewUrl + "2:" + colIndexDataViewUrl, sFilScore, colIndexFilScoreViewUrl + "2:" + colIndexFilScoreViewUrl);
|
||||
}
|
||||
|
||||
function getSumUniqueDailyVisitorsLast30() {
|
||||
getDownloadsFromPiwikApi();
|
||||
moveColumn(sPiwikApiTemp, 10, 11);
|
||||
copyPiwikDownloadsToFilScore();
|
||||
}
|
||||
|
||||
function getDownloadsFromPiwikApi() {
|
||||
var colRangePiwikApiTemp = sPiwikApiTemp.getRange("A1:K");
|
||||
colRangePiwikApiTemp.clearContent();
|
||||
sPiwikApiTemp.getRange("A1").setValue("=ImportJson(CONCATENATE(\"" + piwikUrl + "?module=API&method=Actions.getDownloads&idSite=3&period=range&date=last30&expanded=1&format=json&token_auth=" + piwikApiToken + "\"), \"\", \"\")");
|
||||
var colDataPiwikApiTemp = colRangePiwikApiTemp.getDisplayValues()
|
||||
colRangePiwikApiTemp.clearContent();
|
||||
colRangePiwikApiTemp.setValues(colDataPiwikApiTemp);
|
||||
}
|
||||
|
||||
function copyPiwikDownloadsToFilScore() {
|
||||
var colIndexFilScoreSudv = getColIndex(sFilScore, "sumUniqueDailyVisitorsLast30");
|
||||
var colTopFilScoreSudv = sFilScore.getRange(colIndexFilScoreSudv + "2");
|
||||
var colRangeFilScoreSudv = sFilScore.getRange(colIndexFilScoreSudv + "2:" + colIndexFilScoreSudv);
|
||||
colRangeFilScoreSudv.clearContent();
|
||||
var colIndexPiwikApiTempSudv = getColIndex(sPiwikApiTemp, "Subtablesumdailynbuniqvisitors");
|
||||
var colIndexPiwikApiTempViewUrl = getColIndex(sPiwikApiTemp, "Subtableurl");
|
||||
colTopFilScoreSudv.setValue("=ARRAYFORMULA(IF(ISBLANK($B2:B),\"\",IFERROR(VLOOKUP($B2:B,PiwikApiTemp!$" + colIndexPiwikApiTempSudv + "$2:$" + colIndexPiwikApiTempViewUrl + ",2,FALSE),0)))");
|
||||
var colDataFilScoreSudv = colRangeFilScoreSudv.getDisplayValues();
|
||||
colRangeFilScoreSudv.clearContent();
|
||||
colRangeFilScoreSudv.setValues(colDataFilScoreSudv);
|
||||
}
|
||||
|
||||
function calculateSumUniqueDailyVisitorsLast30Score() {
|
||||
var colIndexFilScoreSudvScore = getColIndex(sFilScore, "sumUniqueDailyVisitorsLast30Score");
|
||||
var colRangeFilScoreSudvScore = sFilScore.getRange(colIndexFilScoreSudvScore + "2:" + colIndexFilScoreSudvScore);
|
||||
colRangeFilScoreSudvScore.clearContent();
|
||||
var colIndexFilScoreSudv = getColIndex(sFilScore, "sumUniqueDailyVisitorsLast30");
|
||||
sFilScore.getRange(colIndexFilScoreSudvScore + "2").setValue("=ARRAYFORMULA(IF(ISBLANK($" + colIndexFilScoreSudv + "2:" + colIndexFilScoreSudv + "),\"\",PERCENTRANK($" + colIndexFilScoreSudv + "$2:$" + colIndexFilScoreSudv + ",$" + colIndexFilScoreSudv + "2:$" + colIndexFilScoreSudv + ")))");
|
||||
var colDataFilScoreSudvScore = colRangeFilScoreSudvScore.getDisplayValues();
|
||||
colRangeFilScoreSudvScore.clearContent();
|
||||
colRangeFilScoreSudvScore.setValues(colDataFilScoreSudvScore);
|
||||
}
|
||||
|
||||
function calculateFilScore() {
|
||||
var filScoreArrayFormula = "=ARRAYFORMULA(";
|
||||
for (var i = 0, filScoreWeightLength = filScoreWeight.length; i < filScoreWeightLength; i++) {
|
||||
var colIndex = getColIndex(sFilScore, filScoreWeight[i][0] + "Score");
|
||||
if (i > 0) {
|
||||
filScoreArrayFormula += "*";
|
||||
}
|
||||
filScoreArrayFormula += "(IF(ISBLANK($" + colIndex + "2:$" + colIndex + "),\"\"," + filScoreWeight[i][1] + "*$" + colIndex + "2:$" + colIndex + "))";
|
||||
}
|
||||
filScoreArrayFormula += ")";
|
||||
updateFilScoreArrayFormula(filScoreArrayFormula)
|
||||
}
|
||||
|
||||
function updateFilScoreArrayFormula(filScoreArrayFormula) {
|
||||
var colIndexFilScoreFilScore = getColIndex(sFilScore, "filScore");
|
||||
var colRangeFilScoreFilScore = sFilScore.getRange(colIndexFilScoreFilScore + "2:" + colIndexFilScoreFilScore);
|
||||
colRangeFilScoreFilScore.clearContent();
|
||||
sFilScore.getRange(colIndexFilScoreFilScore + "2").setValue(filScoreArrayFormula);
|
||||
var colDataFilScoreFilScore = colRangeFilScoreFilScore.getDisplayValues();
|
||||
colRangeFilScoreFilScore.clearContent();
|
||||
colRangeFilScoreFilScore.setValues(colDataFilScoreFilScore);
|
||||
}
|
||||
|
||||
function copyFilScoreToPublic() {
|
||||
var colIndexFilScoreFilScore = getColIndex(sFilScore, "filScore");
|
||||
var colIndexPublicFilScore = getColIndex(sPublic, "filScore");
|
||||
copyRange(sFilScore, colIndexFilScoreFilScore + "2:" + colIndexFilScoreFilScore, sPublic, colIndexPublicFilScore + "2:" + colIndexPublicFilScore);
|
||||
}
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
function createHtmlArrayLinkButton(htmlType, url, listName) {
|
||||
if (url instanceof Array) {
|
||||
for (var i in url) {
|
||||
url[i] = createHtmlArrayLinkButton(htmlType, url[i], listName[i]);
|
||||
}
|
||||
return url;
|
||||
} else {
|
||||
if (url && url != "#N/A") {
|
||||
switch (htmlType) {
|
||||
case "viewUrl":
|
||||
return "<a href=\"" + url + "\" title=\"Preview " + listName + " in your browser.\" class=\"button piwik_download\">View</a>";
|
||||
break;
|
||||
case "addUrl":
|
||||
return "<a href=\"abp:subscribe?location=" + encodeURIComponent(url) + "&title=" + encodeURIComponent(listName) + "\" title=\"Subscribe to " + listName + " in your content blocker.\" class=\"button piwik_download\">Add</a>";
|
||||
break;
|
||||
case "homeUrl":
|
||||
return "<a href=\"" + url + "\" title=\"" + listName + " Home Page\" class=\"button\">Home</a>";
|
||||
break;
|
||||
case "forumUrl":
|
||||
return "<a href=\"" + url + "\" title=\"" + listName + " Forum\" class=\"button\">Forum</a>";
|
||||
break;
|
||||
case "issuesUrl":
|
||||
return "<a href=\"" + url + "\" title=\"" + listName + " GitHub Issues\" class=\"button\">Issues</a>";
|
||||
break;
|
||||
case "email":
|
||||
return "<a href=\"mailto:" + url + "\" title=\"Contribute to " + listName + " via email.\" class=\"button\">Email</a>";
|
||||
break;
|
||||
case "donateUrl":
|
||||
return "<a href=\"" + url + "\" title=\"Donate to support the maintainers of " + listName + ".\" class=\"button\">Donate</a>";
|
||||
break;
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createHtmlArrayTags() {
|
||||
var tagsHtml = [];
|
||||
var numLists = getNumLists(sData);
|
||||
var colIndicesDataTags = getColIndicesWithHeaderSubstring(sData, "tag");
|
||||
for (var i = 0, colIndicesDataTagsLength = colIndicesDataTags.length; i < colIndicesDataTagsLength; i++) {
|
||||
var colTopDataDataTag = sData.getRange(colIndicesDataTags[i] + "1").getValue();
|
||||
var colDataDataTag = sData.getRange(colIndicesDataTags[i] + "2:" + colIndicesDataTags[i]).getValues();
|
||||
for (var j = 0; j < numLists; j++) {
|
||||
if (i == 0) {
|
||||
tagsHtml[j] = "";
|
||||
}
|
||||
if (colDataDataTag[j].toString() != "") {
|
||||
var colDataDataTagParsed = colDataDataTag[j].toString().split(',');
|
||||
for (var k = 0, colDataDataTagParsedLength = colDataDataTagParsed.length; k < colDataDataTagParsedLength; k++) {
|
||||
switch (colTopDataDataTag) {
|
||||
case "tagLang":
|
||||
tagsHtml[j] += "<img src=\"https://filterlists.com/tags/lang-" + colDataDataTagParsed[k] + "-red.svg\" alt=\"lang: " + colDataDataTagParsed[k] + "\">";
|
||||
break;
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return tagsHtml;
|
||||
}
|
||||
|
||||
function createHtmlArrayListName(list) {
|
||||
if (list instanceof Array) {
|
||||
for (var i in list) {
|
||||
list[i] = createHtmlArrayListName(list[i]);
|
||||
}
|
||||
return list;
|
||||
} else {
|
||||
if (list) {
|
||||
return "<h3 id=\"" + trimUrlSlugArray(list) + "\">" + list + "</h3>";
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createHtmlArrayRelatedLists(related) {
|
||||
if (related instanceof Array) {
|
||||
for (var i in related) {
|
||||
related[i] = createHtmlArrayRelatedLists(related[i]);
|
||||
}
|
||||
return related;
|
||||
} else {
|
||||
if (!related) {
|
||||
return "";
|
||||
} else {
|
||||
var relatedArray = related.split(',');
|
||||
var relatedHtml = "<p>";
|
||||
for (var i = 0, relatedArrayLength = relatedArray.length; i < relatedArrayLength; i++) {
|
||||
relatedHtml += "<a href=\"#" + trimUrlSlugArray(relatedArray[i]) + "\" class=\"related\"><span class=\"related\">" + relatedArray[i] + "</span></a>";
|
||||
}
|
||||
relatedHtml += "</p>";
|
||||
return relatedHtml;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,472 +0,0 @@
|
|||
// forked from https://github.com/fastfedora/google-docs
|
||||
/*====================================================================================================================================*
|
||||
ImportJSON by Trevor Lohrbeer (@FastFedora)
|
||||
====================================================================================================================================
|
||||
Version: 1.2.1
|
||||
Project Page: http://blog.fastfedora.com/projects/import-json
|
||||
Copyright: (c) 2012-2013 by Trevor Lohrbeer
|
||||
License: GNU General Public License, version 3 (GPL-3.0)
|
||||
http://www.opensource.org/licenses/gpl-3.0.html
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
A library for importing JSON feeds into Google spreadsheets. Functions include:
|
||||
|
||||
ImportJSON For use by end users to import a JSON feed from a URL
|
||||
ImportJSONViaPost For use by end users to import a JSON feed from a URL using POST parameters
|
||||
ImportJSONAdvanced For use by script developers to easily extend the functionality of this library
|
||||
|
||||
Future enhancements may include:
|
||||
|
||||
- Support for a real XPath like syntax similar to ImportXML for the query parameter
|
||||
- Support for OAuth authenticated APIs (see AddOAuthService__ function for failed experiment)
|
||||
|
||||
Or feel free to write these and add on to the library yourself!
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
Changelog:
|
||||
|
||||
1.2.1 Fixed a bug with how nested arrays are handled. The rowIndex counter wasn't incrementing properly when parsing.
|
||||
1.2.0 Added ImportJSONViaPost and support for fetchOptions to ImportJSONAdvanced
|
||||
1.1.1 Added a version number using Google Scripts Versioning so other developers can use the library
|
||||
1.1 Added support for the noHeaders option
|
||||
1.0 Initial release
|
||||
*====================================================================================================================================*/
|
||||
/**
|
||||
* Imports a JSON feed and returns the results to be inserted into a Google Spreadsheet. The JSON feed is flattened to create
|
||||
* a two-dimensional array. The first row contains the headers, with each column header indicating the path to that data in
|
||||
* the JSON feed. The remaining rows contain the data.
|
||||
*
|
||||
* By default, data gets transformed so it looks more like a normal data import. Specifically:
|
||||
*
|
||||
* - Data from parent JSON elements gets inherited to their child elements, so rows representing child elements contain the values
|
||||
* of the rows representing their parent elements.
|
||||
* - Values longer than 256 characters get truncated.
|
||||
* - Headers have slashes converted to spaces, common prefixes removed and the resulting text converted to title case.
|
||||
*
|
||||
* To change this behavior, pass in one of these values in the options parameter:
|
||||
*
|
||||
* noInherit: Don't inherit values from parent elements
|
||||
* noTruncate: Don't truncate values
|
||||
* rawHeaders: Don't prettify headers
|
||||
* noHeaders: Don't include headers, only the data
|
||||
* debugLocation: Prepend each value with the row & column it belongs in
|
||||
*
|
||||
* For example:
|
||||
*
|
||||
* =ImportJSON("http://gdata.youtube.com/feeds/api/standardfeeds/most_popular?v=2&alt=json", "/feed/entry/title,/feed/entry/content",
|
||||
* "noInherit,noTruncate,rawHeaders")
|
||||
*
|
||||
* @param {url} the URL to a public JSON feed
|
||||
* @param {query} a comma-separated list of paths to import. Any path starting with one of these paths gets imported.
|
||||
* @param {parseOptions} a comma-separated list of options that alter processing of the data
|
||||
*
|
||||
* @return a two-dimensional array containing the data, with the first row containing headers
|
||||
**/
|
||||
function ImportJSON(url, query, parseOptions) {
|
||||
return ImportJSONAdvanced(url, null, query, parseOptions, includeXPath_, defaultTransform_);
|
||||
}
|
||||
/**
|
||||
* Imports a JSON feed via a POST request and returns the results to be inserted into a Google Spreadsheet. The JSON feed is
|
||||
* flattened to create a two-dimensional array. The first row contains the headers, with each column header indicating the path to
|
||||
* that data in the JSON feed. The remaining rows contain the data.
|
||||
*
|
||||
* To retrieve the JSON, a POST request is sent to the URL and the payload is passed as the content of the request using the content
|
||||
* type "application/x-www-form-urlencoded". If the fetchOptions define a value for "method", "payload" or "contentType", these
|
||||
* values will take precedent. For example, advanced users can use this to make this function pass XML as the payload using a GET
|
||||
* request and a content type of "application/xml; charset=utf-8". For more information on the available fetch options, see
|
||||
* https://developers.google.com/apps-script/reference/url-fetch/url-fetch-app . At this time the "headers" option is not supported.
|
||||
*
|
||||
* By default, the returned data gets transformed so it looks more like a normal data import. Specifically:
|
||||
*
|
||||
* - Data from parent JSON elements gets inherited to their child elements, so rows representing child elements contain the values
|
||||
* of the rows representing their parent elements.
|
||||
* - Values longer than 256 characters get truncated.
|
||||
* - Headers have slashes converted to spaces, common prefixes removed and the resulting text converted to title case.
|
||||
*
|
||||
* To change this behavior, pass in one of these values in the options parameter:
|
||||
*
|
||||
* noInherit: Don't inherit values from parent elements
|
||||
* noTruncate: Don't truncate values
|
||||
* rawHeaders: Don't prettify headers
|
||||
* noHeaders: Don't include headers, only the data
|
||||
* debugLocation: Prepend each value with the row & column it belongs in
|
||||
*
|
||||
* For example:
|
||||
*
|
||||
* =ImportJSON("http://gdata.youtube.com/feeds/api/standardfeeds/most_popular?v=2&alt=json", "user=bob&apikey=xxxx",
|
||||
* "validateHttpsCertificates=false", "/feed/entry/title,/feed/entry/content", "noInherit,noTruncate,rawHeaders")
|
||||
*
|
||||
* @param {url} the URL to a public JSON feed
|
||||
* @param {payload} the content to pass with the POST request; usually a URL encoded list of parameters separated by ampersands
|
||||
* @param {fetchOptions} a comma-separated list of options used to retrieve the JSON feed from the URL
|
||||
* @param {query} a comma-separated list of paths to import. Any path starting with one of these paths gets imported.
|
||||
* @param {parseOptions} a comma-separated list of options that alter processing of the data
|
||||
*
|
||||
* @return a two-dimensional array containing the data, with the first row containing headers
|
||||
**/
|
||||
function ImportJSONViaPost(url, payload, fetchOptions, query, parseOptions) {
|
||||
var postOptions = parseToObject_(fetchOptions);
|
||||
if (postOptions["method"] == null) {
|
||||
postOptions["method"] = "POST";
|
||||
}
|
||||
if (postOptions["payload"] == null) {
|
||||
postOptions["payload"] = payload;
|
||||
}
|
||||
if (postOptions["contentType"] == null) {
|
||||
postOptions["contentType"] = "application/x-www-form-urlencoded";
|
||||
}
|
||||
convertToBool_(postOptions, "validateHttpsCertificates");
|
||||
convertToBool_(postOptions, "useIntranet");
|
||||
convertToBool_(postOptions, "followRedirects");
|
||||
convertToBool_(postOptions, "muteHttpExceptions");
|
||||
return ImportJSONAdvanced(url, postOptions, query, parseOptions, includeXPath_, defaultTransform_);
|
||||
}
|
||||
/**
|
||||
* An advanced version of ImportJSON designed to be easily extended by a script. This version cannot be called from within a
|
||||
* spreadsheet.
|
||||
*
|
||||
* Imports a JSON feed and returns the results to be inserted into a Google Spreadsheet. The JSON feed is flattened to create
|
||||
* a two-dimensional array. The first row contains the headers, with each column header indicating the path to that data in
|
||||
* the JSON feed. The remaining rows contain the data.
|
||||
*
|
||||
* The fetchOptions can be used to change how the JSON feed is retrieved. For instance, the "method" and "payload" options can be
|
||||
* set to pass a POST request with post parameters. For more information on the available parameters, see
|
||||
* https://developers.google.com/apps-script/reference/url-fetch/url-fetch-app .
|
||||
*
|
||||
* Use the include and transformation functions to determine what to include in the import and how to transform the data after it is
|
||||
* imported.
|
||||
*
|
||||
* For example:
|
||||
*
|
||||
* ImportJSON("http://gdata.youtube.com/feeds/api/standardfeeds/most_popular?v=2&alt=json",
|
||||
* new Object() { "method" : "post", "payload" : "user=bob&apikey=xxxx" },
|
||||
* "/feed/entry",
|
||||
* "",
|
||||
* function (query, path) { return path.indexOf(query) == 0; },
|
||||
* function (data, row, column) { data[row][column] = data[row][column].toString().substr(0, 100); } )
|
||||
*
|
||||
* In this example, the import function checks to see if the path to the data being imported starts with the query. The transform
|
||||
* function takes the data and truncates it. For more robust versions of these functions, see the internal code of this library.
|
||||
*
|
||||
* @param {url} the URL to a public JSON feed
|
||||
* @param {fetchOptions} an object whose properties are options used to retrieve the JSON feed from the URL
|
||||
* @param {query} the query passed to the include function
|
||||
* @param {parseOptions} a comma-separated list of options that may alter processing of the data
|
||||
* @param {includeFunc} a function with the signature func(query, path, options) that returns true if the data element at the given path
|
||||
* should be included or false otherwise.
|
||||
* @param {transformFunc} a function with the signature func(data, row, column, options) where data is a 2-dimensional array of the data
|
||||
* and row & column are the current row and column being processed. Any return value is ignored. Note that row 0
|
||||
* contains the headers for the data, so test for row==0 to process headers only.
|
||||
*
|
||||
* @return a two-dimensional array containing the data, with the first row containing headers
|
||||
**/
|
||||
function ImportJSONAdvanced(url, fetchOptions, query, parseOptions, includeFunc, transformFunc) {
|
||||
var jsondata = UrlFetchApp.fetch(url, fetchOptions);
|
||||
var object = JSON.parse(jsondata.getContentText());
|
||||
return parseJSONObject_(object, query, parseOptions, includeFunc, transformFunc);
|
||||
}
|
||||
/**
|
||||
* Encodes the given value to use within a URL.
|
||||
*
|
||||
* @param {value} the value to be encoded
|
||||
*
|
||||
* @return the value encoded using URL percent-encoding
|
||||
*/
|
||||
function URLEncode(value) {
|
||||
return encodeURIComponent(value.toString());
|
||||
}
|
||||
/**
|
||||
* Adds an oAuth service using the given name and the list of properties.
|
||||
*
|
||||
* @note This method is an experiment in trying to figure out how to add an oAuth service without having to specify it on each
|
||||
* ImportJSON call. The idea was to call this method in the first cell of a spreadsheet, and then use ImportJSON in other
|
||||
* cells. This didn't work, but leaving this in here for further experimentation later.
|
||||
*
|
||||
* The test I did was to add the following into the A1:
|
||||
*
|
||||
* =AddOAuthService("twitter", "https://api.twitter.com/oauth/access_token",
|
||||
* "https://api.twitter.com/oauth/request_token", "https://api.twitter.com/oauth/authorize",
|
||||
* "<my consumer key>", "<my consumer secret>", "", "")
|
||||
*
|
||||
* Information on obtaining a consumer key & secret for Twitter can be found at https://dev.twitter.com/docs/auth/using-oauth
|
||||
*
|
||||
* Then I added the following into A2:
|
||||
*
|
||||
* =ImportJSONViaPost("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=fastfedora&count=2", "",
|
||||
* "oAuthServiceName=twitter,oAuthUseToken=always", "/", "")
|
||||
*
|
||||
* I received an error that the "oAuthServiceName" was not a valid value. [twl 18.Apr.13]
|
||||
*/
|
||||
function AddOAuthService__(name, accessTokenUrl, requestTokenUrl, authorizationUrl, consumerKey, consumerSecret, method, paramLocation) {
|
||||
var oAuthConfig = UrlFetchApp.addOAuthService(name);
|
||||
if (accessTokenUrl != null && accessTokenUrl.length > 0) {
|
||||
oAuthConfig.setAccessTokenUrl(accessTokenUrl);
|
||||
}
|
||||
if (requestTokenUrl != null && requestTokenUrl.length > 0) {
|
||||
oAuthConfig.setRequestTokenUrl(requestTokenUrl);
|
||||
}
|
||||
if (authorizationUrl != null && authorizationUrl.length > 0) {
|
||||
oAuthConfig.setAuthorizationUrl(authorizationUrl);
|
||||
}
|
||||
if (consumerKey != null && consumerKey.length > 0) {
|
||||
oAuthConfig.setConsumerKey(consumerKey);
|
||||
}
|
||||
if (consumerSecret != null && consumerSecret.length > 0) {
|
||||
oAuthConfig.setConsumerSecret(consumerSecret);
|
||||
}
|
||||
if (method != null && method.length > 0) {
|
||||
oAuthConfig.setMethod(method);
|
||||
}
|
||||
if (paramLocation != null && paramLocation.length > 0) {
|
||||
oAuthConfig.setParamLocation(paramLocation);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Parses a JSON object and returns a two-dimensional array containing the data of that object.
|
||||
*/
|
||||
function parseJSONObject_(object, query, options, includeFunc, transformFunc) {
|
||||
var headers = new Array();
|
||||
var data = new Array();
|
||||
if (query && !Array.isArray(query) && query.toString().indexOf(",") != -1) {
|
||||
query = query.toString().split(",");
|
||||
}
|
||||
if (options) {
|
||||
options = options.toString().split(",");
|
||||
}
|
||||
parseData_(headers, data, "", {
|
||||
rowIndex: 1
|
||||
}, object, query, options, includeFunc);
|
||||
parseHeaders_(headers, data);
|
||||
transformData_(data, options, transformFunc);
|
||||
return hasOption_(options, "noHeaders") ? (data.length > 1 ? data.slice(1) : new Array()) : data;
|
||||
}
|
||||
/**
|
||||
* Parses the data contained within the given value and inserts it into the data two-dimensional array starting at the rowIndex.
|
||||
* If the data is to be inserted into a new column, a new header is added to the headers array. The value can be an object,
|
||||
* array or scalar value.
|
||||
*
|
||||
* If the value is an object, it's properties are iterated through and passed back into this function with the name of each
|
||||
* property extending the path. For instance, if the object contains the property "entry" and the path passed in was "/feed",
|
||||
* this function is called with the value of the entry property and the path "/feed/entry".
|
||||
*
|
||||
* If the value is an array containing other arrays or objects, each element in the array is passed into this function with
|
||||
* the rowIndex incremeneted for each element.
|
||||
*
|
||||
* If the value is an array containing only scalar values, those values are joined together and inserted into the data array as
|
||||
* a single value.
|
||||
*
|
||||
* If the value is a scalar, the value is inserted directly into the data array.
|
||||
*/
|
||||
function parseData_(headers, data, path, state, value, query, options, includeFunc) {
|
||||
var dataInserted = false;
|
||||
if (Array.isArray(value) && isObjectArray_(value)) {
|
||||
for (var i = 0; i < value.length; i++) {
|
||||
if (parseData_(headers, data, path, state, value[i], query, options, includeFunc)) {
|
||||
dataInserted = true;
|
||||
if (data[state.rowIndex]) {
|
||||
state.rowIndex++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (isObject_(value)) {
|
||||
for (key in value) {
|
||||
if (parseData_(headers, data, path + "/" + key, state, value[key], query, options, includeFunc)) {
|
||||
dataInserted = true;
|
||||
}
|
||||
}
|
||||
} else if (!includeFunc || includeFunc(query, path, options)) {
|
||||
// Handle arrays containing only scalar values
|
||||
if (Array.isArray(value)) {
|
||||
value = value.join();
|
||||
}
|
||||
// Insert new row if one doesn't already exist
|
||||
if (!data[state.rowIndex]) {
|
||||
data[state.rowIndex] = new Array();
|
||||
}
|
||||
// Add a new header if one doesn't exist
|
||||
if (!headers[path] && headers[path] != 0) {
|
||||
headers[path] = Object.keys(headers).length;
|
||||
}
|
||||
// Insert the data
|
||||
data[state.rowIndex][headers[path]] = value;
|
||||
dataInserted = true;
|
||||
}
|
||||
return dataInserted;
|
||||
}
|
||||
/**
|
||||
* Parses the headers array and inserts it into the first row of the data array.
|
||||
*/
|
||||
function parseHeaders_(headers, data) {
|
||||
data[0] = new Array();
|
||||
for (key in headers) {
|
||||
data[0][headers[key]] = key;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Applies the transform function for each element in the data array, going through each column of each row.
|
||||
*/
|
||||
function transformData_(data, options, transformFunc) {
|
||||
for (var i = 0; i < data.length; i++) {
|
||||
for (var j = 0; j < data[i].length; j++) {
|
||||
transformFunc(data, i, j, options);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns true if the given test value is an object; false otherwise.
|
||||
*/
|
||||
function isObject_(test) {
|
||||
return Object.prototype.toString.call(test) === '[object Object]';
|
||||
}
|
||||
/**
|
||||
* Returns true if the given test value is an array containing at least one object; false otherwise.
|
||||
*/
|
||||
function isObjectArray_(test) {
|
||||
for (var i = 0; i < test.length; i++) {
|
||||
if (isObject_(test[i])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Returns true if the given query applies to the given path.
|
||||
*/
|
||||
function includeXPath_(query, path, options) {
|
||||
if (!query) {
|
||||
return true;
|
||||
} else if (Array.isArray(query)) {
|
||||
for (var i = 0; i < query.length; i++) {
|
||||
if (applyXPathRule_(query[i], path, options)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return applyXPathRule_(query, path, options);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
/**
|
||||
* Returns true if the rule applies to the given path.
|
||||
*/
|
||||
function applyXPathRule_(rule, path, options) {
|
||||
return path.indexOf(rule) == 0;
|
||||
}
|
||||
/**
|
||||
* By default, this function transforms the value at the given row & column so it looks more like a normal data import. Specifically:
|
||||
*
|
||||
* - Data from parent JSON elements gets inherited to their child elements, so rows representing child elements contain the values
|
||||
* of the rows representing their parent elements.
|
||||
* - Values longer than 256 characters get truncated.
|
||||
* - Values in row 0 (headers) have slashes converted to spaces, common prefixes removed and the resulting text converted to title
|
||||
* case.
|
||||
*
|
||||
* To change this behavior, pass in one of these values in the options parameter:
|
||||
*
|
||||
* noInherit: Don't inherit values from parent elements
|
||||
* noTruncate: Don't truncate values
|
||||
* rawHeaders: Don't prettify headers
|
||||
* debugLocation: Prepend each value with the row & column it belongs in
|
||||
*/
|
||||
function defaultTransform_(data, row, column, options) {
|
||||
if (!data[row][column]) {
|
||||
if (row < 2 || hasOption_(options, "noInherit")) {
|
||||
data[row][column] = "";
|
||||
} else {
|
||||
data[row][column] = data[row - 1][column];
|
||||
}
|
||||
}
|
||||
if (!hasOption_(options, "rawHeaders") && row == 0) {
|
||||
if (column == 0 && data[row].length > 1) {
|
||||
removeCommonPrefixes_(data, row);
|
||||
}
|
||||
data[row][column] = toTitleCase_(data[row][column].toString().replace(/[\/\_]/g, ""));
|
||||
} else if (hasOption_(options, "rawHeaders") && row == 0) {
|
||||
data[row][column] = data[row][column].toString().replace(/[\/\_]/g, "");
|
||||
}
|
||||
if (!hasOption_(options, "noTruncate") && data[row][column]) {
|
||||
data[row][column] = data[row][column].toString().substr(0, 256);
|
||||
}
|
||||
if (hasOption_(options, "debugLocation")) {
|
||||
data[row][column] = "[" + row + "," + column + "]" + data[row][column];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* If all the values in the given row share the same prefix, remove that prefix.
|
||||
*/
|
||||
function removeCommonPrefixes_(data, row) {
|
||||
var matchIndex = data[row][0].length;
|
||||
for (var i = 1; i < data[row].length; i++) {
|
||||
matchIndex = findEqualityEndpoint_(data[row][i - 1], data[row][i], matchIndex);
|
||||
if (matchIndex == 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
for (var i = 0; i < data[row].length; i++) {
|
||||
data[row][i] = data[row][i].substring(matchIndex, data[row][i].length);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Locates the index where the two strings values stop being equal, stopping automatically at the stopAt index.
|
||||
*/
|
||||
function findEqualityEndpoint_(string1, string2, stopAt) {
|
||||
if (!string1 || !string2) {
|
||||
return -1;
|
||||
}
|
||||
var maxEndpoint = Math.min(stopAt, string1.length, string2.length);
|
||||
for (var i = 0; i < maxEndpoint; i++) {
|
||||
if (string1.charAt(i) != string2.charAt(i)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return maxEndpoint;
|
||||
}
|
||||
/**
|
||||
* Converts the text to title case.
|
||||
*/
|
||||
function toTitleCase_(text) {
|
||||
if (text == null) {
|
||||
return null;
|
||||
}
|
||||
return text.replace(/\w\S*/g, function(word) {
|
||||
return word.charAt(0).toUpperCase() + word.substr(1).toLowerCase();
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Returns true if the given set of options contains the given option.
|
||||
*/
|
||||
function hasOption_(options, option) {
|
||||
return options && options.indexOf(option) >= 0;
|
||||
}
|
||||
/**
|
||||
* Parses the given string into an object, trimming any leading or trailing spaces from the keys.
|
||||
*/
|
||||
function parseToObject_(text) {
|
||||
var map = new Object();
|
||||
var entries = (text != null && text.trim().length > 0) ? text.toString().split(",") : new Array();
|
||||
for (var i = 0; i < entries.length; i++) {
|
||||
addToMap_(map, entries[i]);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
/**
|
||||
* Parses the given entry and adds it to the given map, trimming any leading or trailing spaces from the key.
|
||||
*/
|
||||
function addToMap_(map, entry) {
|
||||
var equalsIndex = entry.indexOf("=");
|
||||
var key = (equalsIndex != -1) ? entry.substring(0, equalsIndex) : entry;
|
||||
var value = (key.length + 1 < entry.length) ? entry.substring(key.length + 1) : "";
|
||||
map[key.trim()] = value;
|
||||
}
|
||||
/**
|
||||
* Returns the given value as a boolean.
|
||||
*/
|
||||
function toBool_(value) {
|
||||
return value == null ? false : (value.toString().toLowerCase() == "true" ? true : false);
|
||||
}
|
||||
/**
|
||||
* Converts the value for the given key in the given map to a bool.
|
||||
*/
|
||||
function convertToBool_(map, key) {
|
||||
if (map[key] != null) {
|
||||
map[key] = toBool_(map[key]);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
//https://archive.org/help/wayback_api.php
|
||||
//https://developers.google.com/apps-script/guides/services/external
|
||||
//http://momentjs.com/
|
||||
var moment = Moment.load();
|
||||
|
||||
function GetLastUpdated(urlToCheck) {
|
||||
urlToCheck = urlToCheck.replace("http://", "");
|
||||
urlToCheck = urlToCheck.replace("https://", "");
|
||||
var apiCall = 'http://archive.org/wayback/available?url=' + urlToCheck;
|
||||
try {
|
||||
var response = UrlFetchApp.fetch(apiCall);
|
||||
} catch (e) {
|
||||
var sleepTime = Math.floor(Math.random() * 9000) + 1000;
|
||||
Utilities.sleep(sleepTime);
|
||||
var response = UrlFetchApp.fetch(apiCall);
|
||||
}
|
||||
var json = response.getContentText();
|
||||
var data = JSON.parse(json);
|
||||
return moment(data.archived_snapshots.closest.timestamp, "YYYYMMDDhhmmss").fromNow();;
|
||||
}
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
// refresh each type of url with a separate trigger due to Apps Script execution time limitations
|
||||
function triggerRefreshHttpResponses1() {
|
||||
refreshHttpResponses(1);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses2() {
|
||||
refreshHttpResponses(2);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses3() {
|
||||
refreshHttpResponses(3);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses4() {
|
||||
refreshHttpResponses(4);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses5() {
|
||||
refreshHttpResponses(5);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses6() {
|
||||
refreshHttpResponses(6);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses7() {
|
||||
refreshHttpResponses(7);
|
||||
}
|
||||
|
||||
function triggerRefreshHttpResponses8() {
|
||||
refreshHttpResponses(8);
|
||||
}
|
||||
|
||||
function refreshHttpResponses(triggerNum) {
|
||||
var sheetNameData = typeSheetPrefix[ss.getName()] + "Data";
|
||||
var sheetData = ss.getSheetByName(sheetNameData);
|
||||
var sheetNameStatuses = typeSheetPrefix[ss.getName()] + "UrlStatuses";
|
||||
var sheetStatuses = ss.getSheetByName(sheetNameStatuses);
|
||||
copyRange(sData, "A2:A", sHttpResponse, "A2:A");
|
||||
var listDataMax = getNumLists(sheetData)
|
||||
var urlHeaders = findUrlColumnHeaders(sheetData);
|
||||
if (urlHeaders.length >= triggerNum) {
|
||||
var currentUrlType = urlHeaders[triggerNum - 1];
|
||||
var colHeaderData = findColumnHeader(sheetData, currentUrlType);
|
||||
var colHeaderStatuses = findColumnHeader(sheetStatuses, currentUrlType);
|
||||
var rangeData = sheetData.getRange(colHeaderData + "2:" + colHeaderData + (1 + listDataMax));
|
||||
var rangeStatuses = sheetStatuses.getRange(colHeaderStatuses + "2:" + colHeaderStatuses);
|
||||
rangeStatuses.clearContent();
|
||||
for (var i = 2; i <= rangeData.getNumRows() + 1; i++) {
|
||||
var rangeStatus = sheetStatuses.getRange(colHeaderStatuses + i);
|
||||
var url = rangeData.getCell(i - 1, 1).getValue();
|
||||
if (url) {
|
||||
try {
|
||||
var urlStatus = UrlFetchApp.fetch(url).getResponseCode();
|
||||
rangeStatus.setValue(urlStatus);
|
||||
if (urlStatus != 200) {
|
||||
if (!(urlResponseWhitelist.indexOf(url) > -1)) {
|
||||
MailApp.sendEmail(noticeEmailAddress, urlStatus + ": " + url, urlStatus + ": " + url);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
rangeStatus.setValue(err.message);
|
||||
if (!(urlResponseWhitelist.indexOf(url) > -1)) {
|
||||
MailApp.sendEmail(noticeEmailAddress, "Error: " + url, "Error: " + url + "<br>" + err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function copyListNamesToStatuses(sheetData, sheetStatuses) {
|
||||
var sourceRange = sheetData.getRange("A1:A");
|
||||
var targetRange = sheetStatuses.getRange("A1:A");
|
||||
targetRange.clearContent();
|
||||
sourceRange.copyTo(targetRange);
|
||||
}
|
||||
16
gs/ui.gs
16
gs/ui.gs
|
|
@ -1,16 +0,0 @@
|
|||
var ss = SpreadsheetApp.getActiveSpreadsheet();
|
||||
var sData = ss.getSheetByName("Data");
|
||||
var sPiwikApiTemp = ss.getSheetByName("PiwikApiTemp");
|
||||
var sFilScore = ss.getSheetByName("FilScore");
|
||||
var sHttpResponse = ss.getSheetByName("HttpResponse");
|
||||
var sParsed = ss.getSheetByName("Parsed");
|
||||
var sPublic = ss.getSheetByName("Public");
|
||||
|
||||
function onOpen() {
|
||||
SpreadsheetApp.getUi().createMenu('Refresh').addItem('Refresh All', 'refreshAll').addSeparator().addItem('Refresh Data', 'refreshData').addItem('Refresh FilScore', 'refreshFilScores').addToUi();
|
||||
}
|
||||
|
||||
function refreshAll() {
|
||||
refreshData();
|
||||
refreshFilScores();
|
||||
}
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
function fill2dJsonArrayWithNulls(array2d) {
|
||||
for (var i = 0, array2dLength = array2d.length; i < array2dLength; i++) {
|
||||
for (var j = 0, array2dLength2 = array2d[0].length; j < array2dLength2; j++) {
|
||||
if (array2d[i][j] == null) {
|
||||
array2d[i][j] = "";
|
||||
}
|
||||
}
|
||||
}
|
||||
return array2d;
|
||||
}
|
||||
|
||||
function getColIndex(sheet, colHeader) {
|
||||
var headerData = sheet.getRange("A1:1").getValues();
|
||||
for (var i = 0, headerDataLength = headerData[0].length; i < headerDataLength; i++) {
|
||||
if (headerData[0][i] == colHeader) {
|
||||
return String.fromCharCode(65 + i);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getColIndexNum(sheet, colHeader) {
|
||||
var headerData = sheet.getRange("A1:1").getValues();
|
||||
for (var i = 0, headerDataLength = headerData[0].length; i < headerDataLength; i++) {
|
||||
if (headerData[0][i] == colHeader) {
|
||||
return i + 1;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getColIndicesWithHeaderSubstring(sheet, colHeaderSubstring) {
|
||||
var headerData = sheet.getRange("A1:1").getValues();
|
||||
var colIndices = new Array();
|
||||
for (var i = 0, headerDataLength = headerData[0].length; i < headerDataLength; i++) {
|
||||
if (headerData[0][i].indexOf(colHeaderSubstring) != -1) {
|
||||
colIndices.push(String.fromCharCode(65 + i));
|
||||
}
|
||||
}
|
||||
return colIndices;
|
||||
}
|
||||
|
||||
function trimUrlSlugArray(slugArray) {
|
||||
if (slugArray instanceof Array) {
|
||||
for (var i in slugArray) {
|
||||
slugArray[i] = trimUrlSlugArray(slugArray[i]);
|
||||
}
|
||||
return slugArray;
|
||||
} else {
|
||||
if (slugArray) {
|
||||
var trimCharacters = [" ", "\"", ":", "'", "+", "(", ")", "."];
|
||||
for (var i = 0, trimCharactersLength = trimCharacters.length; i < trimCharactersLength; i++) {
|
||||
slugArray = slugArray.split(trimCharacters[i]).join("");
|
||||
}
|
||||
return slugArray;
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getNumLists(sheetData) {
|
||||
var colIndexDataViewUrl = getColIndex(sData, "viewUrl");
|
||||
var colRangeDataViewUrl = sheetData.getRange(colIndexDataViewUrl + "2:" + colIndexDataViewUrl);
|
||||
for (var i = 1, colRangeDataViewUrlLength = colRangeDataViewUrl.getNumRows(); i <= colRangeDataViewUrlLength; i++) {
|
||||
if (!colRangeDataViewUrl.getCell(i, 1).getValue()) {
|
||||
return i - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function copyRange(sheetSource, rangeSource, sheetTarget, rangeTarget) {
|
||||
var targetRangeListNames = sheetTarget.getRange(rangeTarget);
|
||||
targetRangeListNames.clearContent();
|
||||
sheetSource.getRange(rangeSource).copyTo(targetRangeListNames);
|
||||
}
|
||||
// https://stackoverflow.com/questions/21145080/moving-a-column-in-google-spreadsheet/21152273#21152273
|
||||
function moveColumn(sheet, iniCol, finCol) {
|
||||
var dataRange = sheet.getDataRange();
|
||||
var data = arrayMoveColumn(dataRange.getValues(), iniCol - 1, finCol - 1);
|
||||
dataRange.setValues(data);
|
||||
}
|
||||
|
||||
function arrayMoveColumn(data, from, to) {
|
||||
if (!(data instanceof Array && data[0] instanceof Array)) throw new TypeError('need 2d array');
|
||||
if (from >= data[0].length || to >= data[0].length) throw new Error('index out of bounds');
|
||||
for (var row = 0, dataLength = data.length; row < dataLength; row++) {
|
||||
var temp = data[row].splice(from, 1);
|
||||
data[row].splice(to, 0, temp[0]);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
Loading…
Reference in a new issue