Utilities.py

import glob
import os
import tabula as tbl
import pandas as pd
import copy
import LoggingUtilities as logger
from datetime import datetime
from pyPdf import PdfFileReader

def pdfLister(inputDirectory):
    """
    Function which lists the pdfs present in a user defined directory.
    
    Parameters
    ----------
        inputDirectory : str 
            The folder from which the list of PDF files need to be extracted
    
    Returns
    -------
        pdfList : list
            List of names of the pdfs that exist in the input directory
    
    """
    
    matchingString = os.path.join(inputDirectory, "*.pdf")
    pdfList = glob.glob(matchingString)
    
    return pdfList
    

def getNumPages(pdfFile):
    """
    Function which identifies number of pages in a given PDF file.

    Parameters
    ----------
        pdfFile : str 
            The path of the pdf file whose pages are to be determined    
        
    Returns
    -------
        numPages : int
            Number of pages in the PDF file
    
    """
    
    with open(pdfFile,'rb') as file:
        pdf = PdfFileReader(file)
        numberOfPages = pdf.getNumPages()
    
    return numberOfPages
    
def parseFile(filePath, numPages, expectedNumCols, colsToDrop, colRenameMapping, 
              longFormIdCols, longFormValueCols, logFilePath, outputDirectory):

    """
    Function which parses the individual PDF file into the long form CSV 
    required.

    Parameters
    ----------
        filePath : str 
            The path of the pdf file to be parsed
        
        numPages : int
            Number of pages present in the PDF file
        
        colsToDrop : list
            List of column names to be dropped from the table obtained from the 
            PDF file
        
        colRenameMapping : dict
            Dictionary containing the mapping of old column names to the new 
            column names (Refer configuration file for example)
            
        longFormIdCols : dict
            Columns to be retained for identification when converting the table
            from wide form to long form.
            NOTE: THIS WILL NOT CHANGE FOR A GIVEN TABLE
        
        longFormValueCols : dict
            Columns to be converted into the long form from the wide form
            NOTE: THIS WILL NOT CHANGE FOR A GIVEN TABLE
            
        logFilePath : str
            Path of the log file
            
        outputDirectory : str
            Directory where the output files should be stored
        
    Returns
    -------
        None
    
    """
    
    fileConsolidatedTable = pd.DataFrame()

    #SETTING UP ITERATOR TO ITERATE THROUGH EACH PAGE
    for counter in range(0, numPages):   
        try:

            #READING THE RAW TABLE/S ON THE PAGE
            pageNo = counter+1
            pageRawTable = tbl.read_pdf(filePath, pages = pageNo, spreadsheet = True)
            pageRawTable = pageRawTable.replace("\\r", "", regex = True) #TREATING NEW LINE ESCAPE CHARACTERS
            numRows = len(pageRawTable.index)
            numCols = len(pageRawTable.columns)
            
            #TREATING ESCAPE CHARACTER IN COLUMN KHATA/SURVEY
            columnList = pageRawTable.columns
            columnList = [columnName.replace("\r", "") for columnName in columnList]
            pageRawTable.columns = columnList
            
            #HANDLING CONDITION WHERE EXTRA COLUMN MIGHT EXIST DUE TO DATA DISCREPANCIES
            if numCols != expectedNumCols:
                errorMessage = "Expected " + str(expectedNumCols) + " columns, but found " + str(numCols) + " columns"
                logger.logPageError(logFilePath = logFilePath, pdfFile = filePath, 
                                    pageNo = pageNo, errorMessage = errorMessage)
                continue

            #REMOVING THE LAST TWO ROWS, IN CASE THE PAGE IS THE LAST PAGE  
            if pageNo == numPages:
                pageRawTable.drop([numRows-2, numRows-1], inplace = True, axis = 0)

                numRows = len(pageRawTable.index)
                numCols = len(pageRawTable.columns)      
            
            #SETTING UP THE INDICES FOR ROWS WITH APP NO AND ROWS WITH AMT
            appRowIndices = range(0, numRows, 2)
            amtRowIndices = range(1, numRows, 2) #NOTE: WE EXCLUDE THE LAST ROW, AS THE VALUE POSITIONS ARE DIFFERENT IN IT
            lastRowIndex = numRows - 1            
            
            #TREATING DATAFRAME TO REMOVE USER SSPECIFIED COLUMNS
            pageRawTable.iloc[lastRowIndex] = pageRawTable.iloc[lastRowIndex].shift(6, axis = 0) #CORRECTING OFFSET IN LAST ROW
            pageRawTable.drop(colsToDrop, axis = 1, inplace = True)
            longFormIdCols = [colName for colName in longFormIdCols if colName not in colsToDrop]
            
            #SETTING UP DATAFRAMES CONTAINING APP_NO AND AMT
            appNoDataFrame = copy.deepcopy(pageRawTable.iloc[appRowIndices, :])
            appNoDataFrame = appNoDataFrame.reset_index()
            amtDataFrame = copy.deepcopy(pageRawTable.iloc[amtRowIndices, :])
            amtDataFrame = amtDataFrame.reset_index()
            appNoDataFrame.drop("index", axis = 1, inplace = True)
            amtDataFrame.drop("index", axis = 1, inplace = True)        

            amtDataFrame.ix[:,longFormIdCols] = appNoDataFrame.ix[:, longFormIdCols]
            
            #CONVERTING APP_NO DATAFRAME FROM WIDE TO LONG FORMAT
            appNoDataFrame = pd.melt(appNoDataFrame, 
                                     id_vars = longFormIdCols, 
                                     value_vars = longFormValueCols)            


            #RENAMING THE NEW COLUMNS IN APP_NO DATAFRAME
            appNoDataFrame = appNoDataFrame.rename(index = str, columns = {"variable": "year", "value": "app_no"})        
            
            #CONVERTING AMT DATAFRAME FROM WIDE TO LONG FORMAT
            amtDataFrame = pd.melt(amtDataFrame, 
                                     id_vars = longFormIdCols, 
                                     value_vars = longFormValueCols)

            #RENAMING THE NEW COLUMNS IN AMT DATAFRAME
            amtDataFrame = amtDataFrame.rename(index = str, columns = {"variable": "year", "value": "amt"})        
            
            #MERGING THE TWO DATAFRAMES TO GET A CONSOLIDATED FRAME FOR THE PAGE
            joinKeyList = longFormIdCols + ["year"]   
            processedPageTable = appNoDataFrame.merge(amtDataFrame, on = joinKeyList, how = "left")
            
            #RENAMING COLUMNS BEFORE CONVERTING TO WIDE AND LONG FORM
            processedPageTable.rename(colRenameMapping, inplace = True)

            #DROPPING ROWS CORRESPONDING TO YEARS WHERE TAX WASNT PAID
            processedPageTable = processedPageTable.dropna(axis = 0, subset = ["amt", "app_no"])
            #processedPageTable.to_csv(os.path.basename(filePath) + str(pageNo) + ".csv", index = False)
            
            #APPENDING THE PROCESSED DATAFRAME TO THE OVERALL FILE DATAFRAME
            if len(fileConsolidatedTable.index) == 0:
                fileConsolidatedTable = processedPageTable
            else:
                fileConsolidatedTable = fileConsolidatedTable.append(processedPageTable)     

        except Exception as e:
            logger.logPageError(logFilePath = logFilePath, pdfFile = filePath, 
                                pageNo = pageNo, errorMessage = str(e))
            continue
    
    outputFileName = os.path.basename(filePath) + ".csv"
    outputFile = os.path.join(outputDirectory, outputFileName)
    fileConsolidatedTable.to_csv(outputFile, index = False)