~savoy/ade

9d95dcfa0bb87b426b605867a4298532a9a28520 — savoy 2 months ago cfdae67
changed: headcount no longer holds manager changes

The original reporting which uses the headcount is outdated, while any
future use of it or a recreation will demand a change. The use of not
changing managers was specific to that report.

Signed-off-by: savoy <git@liberation.red>
2 files changed, 35 insertions(+), 84 deletions(-)

M ade/lib/connections.py
M ade/lib/local.py
M ade/lib/connections.py => ade/lib/connections.py +0 -3
@@ 223,9 223,6 @@ class Sqlite:
                costCenterCode TEXT,
                hireDate TIMESTAMP,
                jobTitle TEXT,
                hold BOOLEAN NOT NULL,
                pendingM TEXT,
                pendingS TEXT,
                miscode BOOLEAN NOT NULL,
                gone BOOLEAN NOT NULL)""",
                    """

M ade/lib/local.py => ade/lib/local.py +35 -81
@@ 25,7 25,7 @@ import numpy as np
import pandas as pd

from lib import admin, connections
from lib.util import data, sql
from lib.util import data, sql, cleanup


@admin.initializor


@@ 361,7 361,7 @@ def get(
                    e.employeeId, e.employee, e.department, e.costCenterCode,
                    e.hireDate, e.jobTitle, e.managerId, m.employee as manager,
                    e.skipManagerId, s.employee as skipManager,
                    e.hold, e.pendingM, e.pendingS, e.miscode, e.gone
                    e.miscode, e.gone

                    FROM Employees e



@@ 382,9 382,6 @@ def get(
                    "manager": ("m.employee", "string"),
                    "skipManagerId": ("e.skipManagerId", "Int64"),
                    "skipManager": ("s.employee", "string"),
                    "hold": ("e.hold", "bool"),
                    "pendingM": ("e.pendingM", "string"),
                    "pendingS": ("e.pendingS", "string"),
                    "miscode": ("e.miscode", "bool"),
                    "gone": ("e.gone", "bool"),
                },


@@ 398,8 395,7 @@ def get(
                    e.employeeId AS EmployeeID, e.employee AS Employee,
                    e.managerId AS ManagerID, m.employee as Manager,
                    e.skipManagerId AS SkipManagerID, s.employee as SkipManager,
                    e.hold AS Hold, e.pendingM AS PendingM,
                    e.pendingS AS PendingS, e.miscode AS Miscode,
                    e.miscode AS Miscode,
                    e.gone AS Gone, e.department

                    FROM Employees e


@@ 418,9 414,6 @@ def get(
                    "Manager": ("m.employee", "string"),
                    "SkipManagerID": ("e.skipManagerId", "int64"),
                    "SkipManager": ("s.employee", "string"),
                    "Hold": ("e.hold", "bool"),
                    "PendingM": ("e.pendingM", "string"),
                    "PendingS": ("e.pendingS", "string"),
                    "Miscode": ("e.miscode", "bool"),
                    "Gone": ("e.gone", "bool"),
                },


@@ 1115,7 1108,7 @@ class Headcount(pd.core.frame.DataFrame):
        # Creates a working DataFrame of <self> as further merges will create
        # new objects as pd.core.frame.DataFrame instead of sql.Headcount.
        # Also pulls in the newest Managers and SkipManager tables.
        working = self.copy()
        cached_db = self.copy()
        managers: pd.DataFrame = get("headcount", "Managers")
        skips: pd.DataFrame = get("headcount", "SkipManagers")



@@ 1163,7 1156,7 @@ class Headcount(pd.core.frame.DataFrame):
        people = people.astype(
            {
                key: value
                for key, value in working.reset_index().dtypes.to_dict().items()
                for key, value in cached_db.reset_index().dtypes.to_dict().items()
                if key in people.columns
            }
        )


@@ 1172,7 1165,7 @@ class Headcount(pd.core.frame.DataFrame):
        # Includes creation of "total" frame, which merges managers as working
        # in order to include people like Bruce Pool (not a NAM employee) and their
        # IDs so that they don't show as NA.
        emp_ids = working.loc[:, ["employee"]].copy()
        emp_ids = cached_db.loc[:, ["employee"]].copy()
        ids = pd.concat(
            [emp_ids, managers, skips, people.loc[:, ["employeeId", "employee"]]],
            sort=False,


@@ 1191,25 1184,22 @@ class Headcount(pd.core.frame.DataFrame):
            people[f"{emp}Id"] = people[f"{emp}Id"].astype("Int64")

        # New IDs not present in `working`, prepare for appending
        old = people.loc[people.employeeId.isin(working.index)].copy()
        new = people.loc[~people.employeeId.isin(working.index)].copy()
        existing_employees = people.loc[people.employeeId.isin(cached_db.index)].copy()
        new_employees = people.loc[~people.employeeId.isin(cached_db.index)].copy()

        # Works on new employees if there are any
        if not new.empty:
        if not new_employees.empty:
            admin.instance.logger.info("Adding new employees.")
            new["hold"] = False
            new["pendingM"] = " "
            new["pendingS"] = " "

            # Checks manager hierarchy in old table and if one of the new manager pairs
            # is already labeled as Miscoded (ie labeled as a non-tech employee), the
            # coding is mimicked.
            mis = working.loc[
                working.miscode == True, ["managerId", "skipManagerId"]
            mis = cached_db.loc[
                cached_db.miscode == True, ["managerId", "skipManagerId"]
            ].copy()
            new["miscode"] = np.where(
                (new.managerId.isin(mis.managerId))
                & (new.skipManagerId.isin(mis.skipManagerId)),
            new_employees["miscode"] = np.where(
                (new_employees.managerId.isin(mis.managerId))
                & (new_employees.skipManagerId.isin(mis.skipManagerId)),
                True,
                False,
            )


@@ 1218,73 1208,39 @@ class Headcount(pd.core.frame.DataFrame):
                "No new employees to add, skipping to management."
            )

        # Checks if any employee has had a change of manager. If the employee's
        # managers are currently on HOLD, the new pair replaces the old. If they're
        # not, the new pair is inserted as pending and they're placed on HOLD
        # automatically. The new held pairs are printed as reference.
        old["newConcat"] = (
            old.managerId.astype(str)
            + " "
            + old.manager
            + "|"
            + old.skipManagerId.astype(str)
            + " "
            + old.skipManager
        )
        working = working.reset_index().merge(
            old.loc[:, ["employeeId", "newConcat"]],
        # Updates manager and skip managers
        cached_db = cached_db.reset_index().merge(
            existing_employees.loc[
                :,
                ["employeeId", "managerId", "manager", "skipManagerId", "skipManager"],
            ],
            how="left",
            on=["employeeId"],
            validate="1:1",
        )
        working.newConcat.fillna("", inplace=True)
        working["managerConcat"] = (
            working.managerId.astype(str)
            + " "
            + working.manager
            + "|"
            + working.skipManagerId.astype(str)
            + " "
            + working.skipManager
        )

        working["pendingConcat"] = np.where(
            working.newConcat != working.managerConcat, working.newConcat, " | "
        )
        pending = working.pendingConcat.str.extract(
            r"^([0-9A-Za-z ]+)\|([0-9A-Za-z ]+)$"
        )
        pending[2] = np.where(
            pending[0].isin([" ", np.NaN]) | pending[1].isin([" ", np.NaN]), False, True
        )
        working.pendingM = pending[0]
        working.pendingS = pending[1]
        working.hold = pending[2]

        working.drop(
            columns=["newConcat", "pendingConcat", "managerConcat"], inplace=True
        )
        cached_db = data.remove_column_duplicates(cached_db, reverse=True)

        # Combine running working table with the additions from people IF
        # there are new employees.
        try:
            working = pd.concat([working, new], sort=False, ignore_index=True)
            working.sort_values(by=["employeeId"], inplace=True)
            cached_db = pd.concat(
                [cached_db, new_employees], sort=False, ignore_index=True
            )
            cached_db.sort_values(by=["employeeId"], inplace=True)
        except NameError:
            pass
        working.set_index("employeeId", inplace=True)
        cached_db.set_index("employeeId", inplace=True)

        # Marks any working not in the People+ output as no longer with the
        # company.
        working["gone"] = np.where(~working.index.isin(people.employeeId), True, False)
        cached_db["gone"] = np.where(
            ~cached_db.index.isin(people.employeeId), True, False
        )

        # Final cleaning of non-NAM managers with no ID and of dtypes. Also
        # resets <self> with all of the updates in <working>.
        working.pendingM.fillna(" ", inplace=True)
        working.pendingS.fillna(" ", inplace=True)
        for x in ["pendingM", "pendingS"]:
            working[x] = working[x].astype("string")
        self._reset(working)
        cached_db = cleanup.na(cached_db)
        self._reset(cached_db)
        self.create_id(inplace=True)

        with connections.Sqlite("headcount", upload=True) as conn:


@@ 1298,13 1254,12 @@ class Headcount(pd.core.frame.DataFrame):
                """
                INSERT INTO Employees
                (employeeId, employee, hireDate, jobTitle, managerId,
                    skipManagerId, department, costCenterCode, hold,
                    pendingM, pendingS, miscode, gone)
                    skipManagerId, department, costCenterCode,
                    miscode, gone)

                SELECT
                employeeId, employee, hireDate, jobTitle, managerId,
                skipManagerId, department, costCenterCode, hold, pendingM,
                pendingS, miscode, gone
                skipManagerId, department, costCenterCode, miscode, gone
                FROM temporaryEmployees WHERE True

                ON CONFLICT(employeeId)


@@ 1313,8 1268,7 @@ class Headcount(pd.core.frame.DataFrame):
                jobTitle=excluded.jobTitle, managerId=excluded.managerId,
                skipManagerId=excluded.skipManagerId,
                costCenterCode=excluded.costCenterCode,
                department=excluded.department, hold=excluded.hold,
                pendingM=excluded.pendingM, pendingS=excluded.pendingS,
                department=excluded.department,
                miscode=excluded.miscode, gone=excluded.gone
                """
            )