chore(docs.angularjs.org): serve snapshots for googlebot

This commit restores serving the plain partials (content) when a docs page is accessed with ?_escaped_fragment_=. The Google Ajax Crawler accesses these urls when the page has `<meta type="fragment" content="!">` is set. During the migration to Firebase, this was lost, which resulted in Google dropping the docs almost completely from the index. We are using a Firebase cloud function to serve the partials. Since we cannot access the static hosted files from the function, we have to deploy them as part of the function directory instead, from which they can be read. Related to #16432 Related to #16417
2018-02-12 10:49:19 +01:00
parent 5a022524a3
commit 8681fce27b
7 changed files with 4377 additions and 10 deletions
@@ -13,6 +13,8 @@ var semver = require('semver');
 var exec = require('shelljs').exec;
 var pkg = require(__dirname + '/package.json');

+var docsScriptFolder = 'scripts/docs.angularjs.org-firebase';
+
 // Node.js version checks
 if (!semver.satisfies(process.version, pkg.engines.node)) {
  reportOrFail('Invalid node version (' + process.version + '). ' +
@@ -165,7 +167,8 @@ module.exports = function(grunt) {
      tmp: ['tmp'],
      deploy: [
        'deploy/docs',
-        'deploy/code'
+        'deploy/code',
+        docsScriptFolder + '/functions/html'
      ]
    },

@@ -354,7 +357,17 @@ module.exports = function(grunt) {
            src: '**',
            dest: 'deploy/docs/',
            expand: true
-          }
+          },
+          {
+            src: ['build/docs/index-production.html'],
+            dest: docsScriptFolder + '/functions/content',
+            expand: true,
+            flatten: true
+          },
+          {
+            cwd: 'build/docs',
+            src: 'partials/**',
+            dest: docsScriptFolder + '/functions/content',
            expand: true
          }
        ]
@@ -297,12 +297,13 @@ module.exports = {
  // Our Firebase projects are in subfolders, but Travis expects them in the root,
  // so we need to modify the upload folder path and copy the file into the root
  firebaseDocsJsonForTravis: function() {
-    var docsScriptFolder = 'scripts/docs.angularjs.org-firebase/';
+    var docsScriptFolder = 'scripts/docs.angularjs.org-firebase';

-    var fileName = docsScriptFolder + 'firebase.json';
+    var fileName = docsScriptFolder + '/firebase.json';
    var json = grunt.file.readJSON(fileName);

    json.hosting.public = 'deploy/docs';
+    json.functions.source = docsScriptFolder + '/functions';

    grunt.file.write('firebase.json', JSON.stringify(json));
  }
@@ -23,9 +23,14 @@
        "destination": "/index-production.html"
      },
      {
-        "source": "**/*!(.jpg|.jpeg|.gif|.png|.html|.js|.map|.json|.css|.svg|.ttf|.txt|.woff|.woff2|.eot|.xml)",
-        "destination": "/index-production.html"
+        "source": "**/*!(.@(jpg|jpeg|gif|png|html|js|map|json|css|svg|ttf|txt|woff|woff2|eot|xml))",
+        "function": "sendFile"
      }
    ]
+  },
+  "functions": {
+    "predeploy": [
+      "npm --prefix $RESOURCE_DIR run lint"
+    ]
  }
-}
+}
@@ -0,0 +1,49 @@
+'use strict';
+
+const functions = require('firebase-functions');
+const fs = require('fs');
+
+const BROWSER_CACHE_DURATION = 60 * 60;
+const CDN_CACHE_DURATION = 60 * 60 * 12;
+
+const headers = {
+  'Cache-Control': `public max-age=${BROWSER_CACHE_DURATION} s-maxage=${CDN_CACHE_DURATION}`
+};
+
+const buildSnapshot = data => `<!DOCTYPE html>
+  <html lang="en">
+    <head>
+      <meta charset="utf-8" />
+      <meta name="viewport" content="width=device-width, initial-scale=1">
+      <base href="/">
+    </head>
+    <body>
+      ${data}
+    </body>
+  </html>`;
+
+function sendFile(request, response) {
+
+  const snapshotRequested = typeof request.query._escaped_fragment_ !== 'undefined';
+  const filePath = `content/${snapshotRequested ? `partials${request.path}` : 'index-production'}.html`;
+
+  if (snapshotRequested) {
+    fs.readFile(filePath, {encoding: 'utf8'}, (error, data) => {
+      if (error) {
+        response
+          .status(404)
+          .end();
+      } else {
+        response
+          .set(headers)
+          .send(buildSnapshot(data));
+      }
+    });
+  } else {
+    response
+      .set(headers)
+      .sendFile(filePath, {root: __dirname});
+  }
+}
+
+exports.sendFile = functions.https.onRequest(sendFile);
@@ -0,0 +1,21 @@
+{
+  "name": "functions",
+  "description": "Cloud Functions for Firebase",
+  "scripts": {
+    "lint": "./node_modules/.bin/eslint .",
+    "serve": "firebase serve --only functions",
+    "shell": "firebase experimental:functions:shell",
+    "start": "npm run shell",
+    "deploy": "firebase deploy --only functions",
+    "logs": "firebase functions:log"
+  },
+  "dependencies": {
+    "firebase-admin": "~5.8.1",
+    "firebase-functions": "^0.8.1"
+  },
+  "devDependencies": {
+    "eslint": "^4.12.0",
+    "eslint-plugin-promise": "^3.6.0"
+  },
+  "private": true
+}
@@ -1,10 +1,23 @@
 Firebase for docs.angularjs.org
 ===============================

-The docs are deployed to Google Firebase hosting via Travis deployment config, which expects
-firebase.json and .firebaserc in the repository root.
+# Continuous integration

-See travis.yml for the complete deployment config.
+The docs are deployed to Google Firebase hosting via Travis deployment config, which expects
+firebase.json in the repository root, which is done by a Grunt task (firebaseDocsJsonForTravis)
+that modifies the paths in the firebase.json and copies it into the repository root.
+
+See travis.yml for the complete deployment config, and scripts/travis/build.sh for the full deployment
+build steps.
+
+# Serving locally:
+
+- Run `grunt:prepareDeploy`.
+  This copies docs content files into deploy/docs and the partials for Search Engine AJAX
+  Crawling into ./functions/content.
+
+- Run `firebase serve --only functions,hosting`
+  Creates a server at localhost:5000 that serves from deploy/docs and uses the local function

 See /scripts/code.angularjs.org-firebase/readme.firebase.code.md for the firebase deployment to
 code.angularjs.org