Merge pull request #281 from rigrig/data_files

Data files

README.md

@@ -202,6 +202,11 @@ and which gives information about the app for the Launcher.
   "files:"file1,file2,file3",
      // added by BangleApps loader on upload - lists all files
      // that belong to the app so it can be deleted
+  "data":"appid.data.json,appid.data?.json;appidStorageFile,appidStorageFile*"
+     // added by BangleApps loader on upload - lists files that 
+     // the app might write, so they can be deleted on uninstall
+     // typically these files are not uploaded, but created by the app
+     // these can include '*' or '?' wildcards
 }
 ```
 
@@ -240,16 +245,24 @@ and which gives information about the app for the Launcher.
      "evaluate":true          // if supplied, data isn't quoted into a String before upload
                               // (eg it's evaluated as JS)
     },
+  ]
+  "data": [                   // list of files the app writes to
+    {"name":"appid.data.json",  // filename used in storage
+     "storageFile":true       // if supplied, file is treated as storageFile
+    },
+    {"wildcard":"appid.data.*" // wildcard of filenames used in storage
+    },                         // this is mutually exclusive with using "name" 
+  ],
   "sortorder" : 0,            // optional - choose where in the list this goes.
                               // this should only really be used to put system
                               // stuff at the top
-  ]
 }
 ```
 
 * name, icon and description present the app in the app loader.
 * tags is used for grouping apps in the library, separate multiple entries by comma. Known tags are `tool`, `system`, `clock`, `game`, `sound`, `gps`, `widget`, `launcher` or empty.
 * storage is used to identify the app files and how to handle them
+* data is used to clean up files when the app is uninstalled
 
 ### `apps.json`: `custom` element
 
@@ -335,10 +348,10 @@ Example `settings.js`
 ```js
 // make sure to enclose the function in parentheses
 (function(back) {
-  let settings = require('Storage').readJSON('app.settings.json',1)||{};
+  let settings = require('Storage').readJSON('app.json',1)||{};
   function save(key, value) {
     settings[key] = value;
-    require('Storage').write('app.settings.json',settings);
+    require('Storage').write('app.json',settings);
   } 
   const appMenu = {
     '': {'title': 'App Settings'},
@@ -351,19 +364,20 @@ Example `settings.js`
   E.showMenu(appMenu)
 })
 ```
-In this example the app needs to add both `app.settings.js` and
+In this example the app needs to add `app.settings.js` to `storage` in `apps.json`.   
-`app.settings.json` to `apps.json`:
+It should also add `app.json` to `data`, to make sure it is cleaned up when the app is uninstalled.
 ```json
   { "id": "app",
     ...
     "storage": [
       ...
       {"name":"app.settings.js","url":"settings.js"},
-      {"name":"app.settings.json","content":"{}"}
+    ],
+    "data": [
+      {"name":"app.json"}
     ]
   },
 ```
-That way removing the app also cleans up `app.settings.json`.
 
 ## Coding hints
 

bin/sanitycheck.js

@@ -39,10 +39,26 @@ try{
 
 const APP_KEYS = [
   'id', 'name', 'shortName', 'version', 'icon', 'description', 'tags', 'type',
-  'sortorder', 'readme', 'custom', 'interface', 'storage', 'allow_emulator',
+  'sortorder', 'readme', 'custom', 'interface', 'storage', 'data', 'allow_emulator',
 ];
 const STORAGE_KEYS = ['name', 'url', 'content', 'evaluate'];
+const DATA_KEYS = ['name', 'wildcard', 'storageFile'];
+const FORBIDDEN_FILE_NAME_CHARS = /[,;]/; // used as separators in appid.info
 
+function globToRegex(pattern) {
+  const ESCAPE = '.*+-?^${}()|[]\\';
+  const regex = pattern.replace(/./g, c => {
+    switch (c) {
+      case '?': return '.';
+      case '*': return '.*';
+      default: return ESCAPE.includes(c) ? ('\\' + c) : c;
+    }
+  });
+  return new RegExp('^'+regex+'$');
+}
+const isGlob = f => /[?*]/.test(f)
+// All storage+data files in all apps: {app:<appid>,[file:<storage.name> | data:<data.name|data.wildcard>]}
+let allFiles = [];
 apps.forEach((app,appIdx) => {
   if (!app.id) ERROR(`App ${appIdx} has no id`);
   //console.log(`Checking ${app.id}...`);
@@ -74,9 +90,13 @@ apps.forEach((app,appIdx) => {
   var fileNames = [];
   app.storage.forEach((file) => {
     if (!file.name) ERROR(`App ${app.id} has a file with no name`);
+    if (isGlob(file.name)) ERROR(`App ${app.id} storage file ${file.name} contains wildcards`);
+    let char = file.name.match(FORBIDDEN_FILE_NAME_CHARS)
+    if (char) ERROR(`App ${app.id} storage file ${file.name} contains invalid character "${char[0]}"`)
     if (fileNames.includes(file.name))
       ERROR(`App ${app.id} file ${file.name} is a duplicate`);
     fileNames.push(file.name);
+    allFiles.push({app: app.id, file: file.name});
     if (file.url) if (!fs.existsSync(appDir+file.url)) ERROR(`App ${app.id} file ${file.url} doesn't exist`);
     if (!file.url && !file.content && !app.custom) ERROR(`App ${app.id} file ${file.name} has no contents`);
     var fileContents = "";
@@ -115,6 +135,54 @@ apps.forEach((app,appIdx) => {
       if (!STORAGE_KEYS.includes(key)) ERROR(`App ${app.id}'s ${file.name} has unknown key ${key}`);
     }
   });
+  let dataNames = [];
+  (app.data||[]).forEach((data)=>{
+    if (!data.name && !data.wildcard) ERROR(`App ${app.id} has a data file with no name`);
+    if (dataNames.includes(data.name||data.wildcard))
+      ERROR(`App ${app.id} data file ${data.name||data.wildcard} is a duplicate`);
+    dataNames.push(data.name||data.wildcard)
+    allFiles.push({app: app.id, data: (data.name||data.wildcard)});
+    if ('name' in data && 'wildcard' in data)
+      ERROR(`App ${app.id} data file ${data.name} has both name and wildcard`);
+    if (isGlob(data.name))
+      ERROR(`App ${app.id} data file name ${data.name} contains wildcards`);
+    if (data.wildcard) {
+      if (!isGlob(data.wildcard))
+        ERROR(`App ${app.id} data file wildcard ${data.wildcard} does not actually contains wildcard`);
+      if (data.wildcard.replace(/\?|\*/g,'') === '')
+        ERROR(`App ${app.id} data file wildcard ${data.wildcard} does not contain regular characters`);
+      else if (data.wildcard.replace(/\?|\*/g,'').length < 3)
+        WARN(`App ${app.id} data file wildcard ${data.wildcard} is very broad`);
+      else if (!data.wildcard.includes(app.id))
+        WARN(`App ${app.id} data file wildcard ${data.wildcard} does not include app ID`);
+    }
+    let char = (data.name||data.wildcard).match(FORBIDDEN_FILE_NAME_CHARS)
+    if (char) ERROR(`App ${app.id} data file ${data.name||data.wildcard} contains invalid character "${char[0]}"`)
+    if ('storageFile' in data && typeof data.storageFile !== 'boolean')
+      ERROR(`App ${app.id} data file ${data.name||data.wildcard} has non-boolean value for "storageFile"`);
+    for (const key in data) {
+      if (!DATA_KEYS.includes(key))
+        ERROR(`App ${app.id} data file ${data.name||data.wildcard} has unknown property "${key}"`);
+    }
+  });
+  // prefer "appid.json" over "appid.settings.json" (TODO: change to ERROR once all apps comply?)
+  if (dataNames.includes(app.id+".settings.json") && !dataNames.includes(app.id+".json"))
+    WARN(`App ${app.id} uses data file ${app.id+'.settings.json'} instead of ${app.id+'.json'}`)
+  // settings files should be listed under data, not storage (TODO: change to ERROR once all apps comply?)
+  if (fileNames.includes(app.id+".settings.json"))
+    WARN(`App ${app.id} uses storage file ${app.id+'.settings.json'} instead of data file`)
+  if (fileNames.includes(app.id+".json"))
+    WARN(`App ${app.id} uses storage file ${app.id+'.json'} instead of data file`)
+  // warn if storage file matches data file of same app
+  dataNames.forEach(dataName=>{
+    const glob = globToRegex(dataName)
+    fileNames.forEach(fileName=>{
+      if (glob.test(fileName)) {
+        if (isGlob(dataName)) WARN(`App ${app.id} storage file ${fileName} matches data wildcard ${dataName}`)
+        else WARN(`App ${app.id} storage file ${fileName} is also listed in data`)
+      }
+    })
+  })
   //console.log(fileNames);
   if (isApp && !fileNames.includes(app.id+".app.js")) ERROR(`App ${app.id} has no entrypoint`);
   if (isApp && !fileNames.includes(app.id+".img")) ERROR(`App ${app.id} has no JS icon`);
@@ -123,3 +191,20 @@ apps.forEach((app,appIdx) => {
     if (!APP_KEYS.includes(key)) ERROR(`App ${app.id} has unknown key ${key}`);
   }
 });
+// Do not allow files from different apps to collide
+let fileA
+while(fileA=allFiles.pop()) {
+  const nameA = (fileA.file||fileA.data),
+    globA = globToRegex(nameA),
+    typeA = fileA.file?'storage':'data'
+  allFiles.forEach(fileB => {
+    const nameB = (fileB.file||fileB.data),
+      globB = globToRegex(nameB),
+      typeB = fileB.file?'storage':'data'
+    if (globA.test(nameB)||globB.test(nameA)) {
+      if (isGlob(nameA)||isGlob(nameB))
+        ERROR(`App ${fileB.app} ${typeB} file ${nameB} matches app ${fileA.app} ${typeB} file ${nameA}`)
+      else ERROR(`App ${fileB.app} ${typeB} file ${nameB} is also listed as ${typeA} file for app ${fileA.app}`)
+    }
+  })
+}

js/appinfo.js

@@ -69,13 +69,48 @@ var AppInfo = {
       var fileList = fileContents.map(storageFile=>storageFile.name);
       fileList.unshift(appJSONName); // do we want this? makes life easier!
       json.files = fileList.join(",");
+      if ('data' in app) {
+        let data = {dataFiles: [], storageFiles: []};
+        // add "data" files to appropriate list
+        app.data.forEach(d=>{
+          if (d.storageFile) data.storageFiles.push(d.name||d.wildcard)
+          else data.dataFiles.push(d.name||d.wildcard)
+        })
+        const dataString = AppInfo.makeDataString(data)
+        if (dataString) json.data = dataString
+      }
       fileContents.push({
         name : appJSONName,
         content : JSON.stringify(json)
       });
       resolve(fileContents);
     });
-  }
+  },
+  // (<appid>.info).data holds filenames of data: both regular and storageFiles
+  // These are stored as:  (note comma vs semicolons)
+  //   "fil1,file2", "file1,file2;storageFileA,storageFileB" or ";storageFileA"
+  /**
+   * Convert appid.info "data" to object with file names/patterns
+   * Passing in undefined works
+   * @param data "data" as stored in appid.info
+   * @returns {{storageFiles:[], dataFiles:[]}}
+   */
+  parseDataString(data) {
+    data = data || '';
+    let [files = [], storage = []] = data.split(';').map(d => d.split(','))
+    return {dataFiles: files, storageFiles: storage}
+  },
+  /**
+   * Convert object with file names/patterns to appid.info "data" string
+   * Passing in an incomplete object will not work
+   * @param data {{storageFiles:[], dataFiles:[]}}
+   * @returns {string} "data" to store in appid.info
+   */
+  makeDataString(data) {
+    if (!data.dataFiles.length && !data.storageFiles.length) { return '' }
+    if (!data.storageFiles.length) { return data.dataFiles.join(',') }
+    return [data.dataFiles.join(','),data.storageFiles.join(',')].join(';')
+  },
 };
 
 if ("undefined"!=typeof module)

js/comms.js

@@ -94,10 +94,29 @@ getInstalledApps : () => {
   });
 },
 removeApp : app => { // expects an appid.info structure (i.e. with `files`)
-  if (app.files === '') return Promise.resolve(); // nothing to erase
+  if (!app.files && !app.data) return Promise.resolve(); // nothing to erase
   Progress.show({title:`Removing ${app.name}`,sticky:true});
-  var cmds = app.files.split(',').map(file=>{
+  let cmds = '\x10const s=require("Storage");\n';
-    return `\x10require("Storage").erase(${toJS(file)});\n`;
+  // remove App files: regular files, exact names only
+  cmds += app.files.split(',').map(file => `\x10s.erase(${toJS(file)});\n`).join("");
+  // remove app Data: (dataFiles and storageFiles)
+  const data = AppInfo.parseDataString(app.data)
+  const isGlob = f => /[?*]/.test(f)
+  //   regular files, can use wildcards
+  cmds += data.dataFiles.map(file => {
+    if (!isGlob(file)) return `\x10s.erase(${toJS(file)});\n`;
+    const regex = new RegExp(globToRegex(file))
+    return `\x10s.list(${regex}).forEach(f=>s.erase(f));\n`;
+  }).join("");
+  //   storageFiles, can use wildcards
+  cmds += data.storageFiles.map(file => {
+    if (!isGlob(file)) return `\x10s.open(${toJS(file)},'r').erase();\n`;
+    // storageFiles have a chunk number appended to their real name
+    const regex = globToRegex(file+'\u0001')
+    // open() doesn't want the chunk number though
+    let cmd = `\x10s.list(${regex}).forEach(f=>s.open(f.substring(0,f.length-1),'r').erase());\n`
+    // using a literal \u0001 char fails (not sure why), so escape it
+    return cmd.replace('\u0001', '\\x01')
   }).join("");
   console.log("removeApp", cmds);
   return Comms.reset().then(new Promise((resolve,reject) => {

js/index.js

@@ -349,6 +349,14 @@ function updateApp(app) {
       .filter(f => f !== app.id + '.info')
       .filter(f => !app.storage.some(s => s.name === f))
       .join(',');
+    let data = AppInfo.parseDataString(remove.data)
+    if ('data' in app) {
+      // only remove data files which are no longer declared in new app version
+      const removeData = (f) => !app.data.some(d => (d.name || d.wildcard)===f)
+      data.dataFiles = data.dataFiles.filter(removeData)
+      data.storageFiles = data.storageFiles.filter(removeData)
+    }
+    remove.data = AppInfo.makeDataString(data)
     return Comms.removeApp(remove);
   }).then(()=>{
     showToast(`Updating ${app.name}...`);

js/utils.js

@@ -8,6 +8,18 @@ function escapeHtml(text) {
   };
   return text.replace(/[&<>"']/g, function(m) { return map[m]; });
 }
+// simple glob to regex conversion, only supports "*" and "?" wildcards
+function globToRegex(pattern) {
+  const ESCAPE = '.*+-?^${}()|[]\\';
+  const regex = pattern.replace(/./g, c => {
+    switch (c) {
+      case '?': return '.';
+      case '*': return '.*';
+      default: return ESCAPE.includes(c) ? ('\\' + c) : c;
+    }
+  });
+  return new RegExp('^'+regex+'$');
+}
 function htmlToArray(collection) {
   return [].slice.call(collection);
 }