~shreyasminocha/rice-dining-api

9e364a11a3b76260faa3e217bf7ce1c7530c7233 — Shreyas Minocha 2 years ago fe0afb1
Generalize daily menu scraping
3 files changed, 44 insertions(+), 21 deletions(-)

M package-lock.json
M package.json
M src/scrape.js
M package-lock.json => package-lock.json +17 -0
@@ 9,6 9,7 @@
      "version": "0.1.0",
      "license": "MIT",
      "dependencies": {
        "camelcase": "^6.3.0",
        "got": "^12.0.1",
        "jsdom": "^19.0.0",
        "koa": "^2.13.4",


@@ 218,6 219,17 @@
        "node": ">=8"
      }
    },
    "node_modules/camelcase": {
      "version": "6.3.0",
      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
      "engines": {
        "node": ">=10"
      },
      "funding": {
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
    "node_modules/clone-response": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",


@@ 1475,6 1487,11 @@
        }
      }
    },
    "camelcase": {
      "version": "6.3.0",
      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="
    },
    "clone-response": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",

M package.json => package.json +1 -0
@@ 10,6 10,7 @@
  "author": "Shreyas Minocha",
  "license": "MIT",
  "dependencies": {
    "camelcase": "^6.3.0",
    "got": "^12.0.1",
    "jsdom": "^19.0.0",
    "koa": "^2.13.4",

M src/scrape.js => src/scrape.js +26 -21
@@ 1,26 1,31 @@
import got from 'got';
import jsdom from 'jsdom';
import camelcase from 'camelcase';

const res = await got('https://web-api3.rice.edu/static/seibel-servery-menu-new.js');
const menuHtml = eval(res.body + 'seibelServeryMenuTemplate()'); // eval bad
const dom = new jsdom.JSDOM(`
	<!DOCTYPE html>
	<html>
	<body>
		${menuHtml}
	</body>
	</html>
`);
const doc = dom.window.document;
async function getDailyMenu(servery) {
	const res = await got(`https://web-api3.rice.edu/static/${servery}-menu-new.js`);
	const menuHtml = eval(res.body + `${camelcase(servery)}MenuTemplate()`); // eval bad
	const dom = new jsdom.JSDOM(`
		<!DOCTYPE html>
		<html>
		<body>
			${menuHtml}
		</body>
		</html>
	`);
	const doc = dom.window.document;

const mealToData = (meal) => ({
	meal: meal.querySelector('.servery-title').textContent,
	menu: Array
		.from(meal.querySelectorAll('.menu-items .mitem'))
		.map((mitem) => mitem.textContent),
});
const meals = Array
	.from(doc.querySelectorAll('.inside-daily-menu .item:not(.legend)'))
	.map(mealToData);
	const mealToData = (meal) => ({
		meal: meal.querySelector('.servery-title').textContent,
		menu: Array
			.from(meal.querySelectorAll('.menu-items .mitem'))
			.map((mitem) => mitem.textContent),
	});
	const meals = Array
		.from(doc.querySelectorAll('.inside-daily-menu .item:not(.legend)'))
		.map(mealToData);

console.log(meals);
	return meals;
}

console.log(await getDailyMenu('seibel-servery'));