~denisebitca/esmeralde

e7ea9a44ad1f0682020701f1fa2881b6cc6f714a — Rafael Bitca 2 years ago 266aa3a
Ajouté code pour l'EDT web crawler
3 files changed, 95 insertions(+), 7 deletions(-)

A src/Crawler.ts
D src/Parser.ts
A src/index.js
A src/Crawler.ts => src/Crawler.ts +61 -0
@@ 0,0 1,61 @@
import { CheerioAPI } from 'cheerio';
interface Group {
    name: string;
    id: number;
}

interface SubGroup {
    name: string;
    id: number;
}

class Crawler{
    private DOM: CheerioAPI
    constructor(dom: CheerioAPI){
        this.DOM = dom;
    }

    public getGroups(): Group[] {
        let groups: Group[] = [];
        this.DOM("select[name='selec_groupe'] option").each((i, elem) => {
            if(this.DOM != null && this.DOM(elem).attr('value') != "TOUS"){
                groups.push({
                    name: this.DOM(elem).text(),
                    id: Number(this.DOM(elem).attr('value'))
                });
            }
        });
        return groups;
    }

    public updateDOMUrl(group: Group, sub?: SubGroup) : string{
        let url = "https://edt.iut-orsay.fr/edt_invite.php?selec_groupe=" + group.id + "&hau=1000&lar=1500";
        if(sub != undefined){
            let url = "https://edt.iut-orsay.fr/edt_invite.php?selec_groupe=" + group.id + "&groupes_multi[]=" + sub.id + "&hau=1000&lar=1500";
        }
        return url;
    }

    public updateDOM(dom: CheerioAPI) : void{
        this.DOM = dom;
    }

    public getSubGroups(): SubGroup[]{
        let subGroups: SubGroup[] = [];
        this.DOM("select[name='groupes_multi[]'] option").each((i, elem) => {
            if(this.DOM != null && this.DOM(elem).attr('value') != "TOUS"){
                subGroups.push({
                    name: this.DOM(elem).text(),
                    id: Number(this.DOM(elem).attr('value'))
                });
            }
        });
        return subGroups;
    }

    public getImage(group: Group, sub: SubGroup) : string | null{
        return this.DOM("img")[0].attribs.src;
    }
}

export default Crawler;
\ No newline at end of file

D src/Parser.ts => src/Parser.ts +0 -7
@@ 1,7 0,0 @@
import cheerio from 'cheerio';

class Parser{
    constructor(public input: string){

    }
}
\ No newline at end of file

A src/index.js => src/index.js +34 -0
@@ 0,0 1,34 @@
import Crawler from "./Crawler";
import cheerio from "cheerio";
import https from "https";

https.get("https://edt.iut-orsay.fr/edt_invite.php", (res) => {
    let data = "";
    res.on("data", (chunk) => {
        data += chunk;
    });
    res.on("end", () => {
        const $ = cheerio.load(data);
        const crawler = new Crawler($);
        let myGroups = crawler.getGroups();
        for(let group of myGroups){
            https.get(crawler.updateDOMUrl(group), (res2) => {
                let data2 = "";
                res2.on("data", (chunk) => {
                    data2 += chunk;
                });
                res2.on("end", () => {
                    const $2 = cheerio.load(data2);
                    crawler.updateDOM($2);
                    console.log("\n");
                    console.log("Groupe : " + group.name);
                    console.log("\n");
                    let subGroups = crawler.getSubGroups(group);
                    for(let subGroup of subGroups){
                        console.log(subGroup.name);
                    }
                });
            });
        }
    });
})
\ No newline at end of file