diff --git a/PAMI/coveragePattern/basic/CMine.py b/PAMI/coveragePattern/basic/CMine.py index 8b1dcde4..3274dcf9 100644 --- a/PAMI/coveragePattern/basic/CMine.py +++ b/PAMI/coveragePattern/basic/CMine.py @@ -58,7 +58,7 @@ from PAMI.coveragePattern.basic import abstract as _ab from typing import List, Dict, Tuple, Set, Union, Any, Generator -from deprecation import deprecated +from deprecated import deprecated class CMine(_ab._coveragePatterns): @@ -241,41 +241,45 @@ def creatingCoverageItems(self) -> Dict[str, List[str]]: tidData = {} self._lno = 0 for transaction in self._Database: - self._lno = self._lno + 1 - for item in transaction[1:]: + for item in transaction: if item not in tidData: tidData[item] = [self._lno] else: tidData[item].append(self._lno) + self._lno = self._lno + 1 coverageTidData = {k: v for k, v in tidData.items() if len(v) / len(self._Database) >= self._minRF} coverageTidData = dict(sorted(coverageTidData.items(), reverse=True, key=lambda x: len(x[1]))) return coverageTidData - def tidToBitset(self,item_set: Dict[str, int]) -> Dict[str, int]: + def tidToBitset(self,item_set: Dict[str, List[int]]) -> Dict[str, int]: """ - This function converts tid list to bitset. + This function converts tid list to bitset. Each item's transactions become one + integer with one bit per transaction of the database. - :param item_set: + :param item_set: coverage items and their tid lists :return: Dictionary :rtype: dict[str,int] """ bitset = {} + size = (self._lno + 7) >> 3 for k,v in item_set.items(): - bitset[k] = 0b1 - bitset[k] = (bitset[k] << int(v[0])) | 0b1 - for i in range(1,len(v)): - diff = int(v[i]) - int(v[i-1]) - bitset[k] = (bitset[k] << diff) | 0b1 - bitset[k] = (bitset[k] << (self._lno - int(v[-1]))) + buffer = bytearray(size) + for tid in v: + buffer[tid >> 3] |= 128 >> (tid & 7) + bitset[k] = int.from_bytes(bytes(buffer), 'big') return bitset - def genPatterns(self,prefix: Tuple[str, int],tidData: List[Tuple[str, int]]) -> None: + def genPatterns(self,prefix: Tuple[str, int],tidData: List[Tuple[str, int]],start: int=0) -> None: """ - This function generate coverage pattern about prefix. - - :param prefix: String - :param tidData: list + This function generate coverage pattern about prefix. The prefix carries the union + of the transactions covered by its items, and it is extended with every later item + whose overlap ratio stays within maxOR (sorted closure property). Coverage support + only grows with more items, so it gates the output but never the growth. + + :param prefix: the pattern and the bitset of the transactions it covers + :param tidData: list of coverage items and their bitsets + :param start: index of the first item eligible to extend the prefix :return: None """ # variables to store coverage item set and @@ -283,16 +287,16 @@ def genPatterns(self,prefix: Tuple[str, int],tidData: List[Tuple[str, int]]) -> # Get the length of tidData length = len(tidData) - for i in range(length): + for i in range(start, length): tid = prefix[1] & tidData[i][1] - tid1 = prefix[1] | tidData[i][1] - andCount = bin(tid).count("1") - 1 - orCount = bin(tid1).count("1") - 1 - if orCount/len(self._Database) >= self._minCS and andCount / len(str(prefix[1])) <= self._maxOR: + andCount = tid.bit_count() + if andCount <= self._maxOR * self._mapSupport[tidData[i][0]]: + tid1 = prefix[1] | tidData[i][1] + orCount = tid1.bit_count() coverageItem_set = item_set + '\t' + tidData[i][0] - if orCount / len(self._Database) >= self._minRF: - self._finalPatterns[coverageItem_set] = andCount - self.genPatterns((coverageItem_set,tid),tidData[i+1:length]) + if orCount / len(self._Database) >= self._minCS: + self._finalPatterns[coverageItem_set] = orCount + self.genPatterns((coverageItem_set,tid1),tidData,i+1) def generateAllPatterns(self,coverageItems: Dict[str, int]) -> None: """ @@ -305,7 +309,7 @@ def generateAllPatterns(self,coverageItems: Dict[str, int]) -> None: length = len(tidData) for i in range(length): #print(i,tidData[i][0]) - self.genPatterns(tidData[i],tidData[i+1:length]) + self.genPatterns(tidData[i],tidData,i+1) @deprecated("It is recommended to use 'mine()' instead of 'mine()' for mining process. Starting from January 2025, 'mine()' will be completely terminated.") def startMine(self) -> None: @@ -324,10 +328,10 @@ def mine(self) -> None: self._minRF = self._convert(self._minRF) self._maxOR = self._convert(self._maxOR) coverageItems = self.creatingCoverageItems() + self._mapSupport = {k: len(v) for k, v in coverageItems.items()} self._finalPatterns = {k: len(v) for k, v in coverageItems.items()} coverageItemsBitset = self.tidToBitset(coverageItems) self.generateAllPatterns(coverageItemsBitset) - self.save('output.txt') self._endTime = _ab._time.time() process = _ab._psutil.Process(_ab._os.getpid()) self._memoryUSS = float() @@ -422,7 +426,6 @@ def printResults(self) -> None: if len(_ab._sys.argv) == 6: _ap = CMine(_ab._sys.argv[1], _ab._sys.argv[3], _ab._sys.argv[4], _ab._sys.argv[5]) _ap.mine() - _ap.mine() print("Total number of coverage Patterns:", len(_ap.getPatterns())) _ap.save(_ab._sys.argv[2]) print("Total Memory in USS:", _ap.getMemoryUSS()) diff --git a/PAMI/fuzzyFrequentPattern/cuda/__init__.py b/PAMI/fuzzyFrequentPattern/cuda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/PAMI/fuzzyFrequentPattern/cuda/abstract.py b/PAMI/fuzzyFrequentPattern/cuda/abstract.py new file mode 100644 index 00000000..5913f5d7 --- /dev/null +++ b/PAMI/fuzzyFrequentPattern/cuda/abstract.py @@ -0,0 +1,167 @@ +# Copyright (C) 2021 Rage Uday Kiran +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from abc import ABC as _ABC, abstractmethod as _abstractmethod +import time as _time +import math as _math +import csv as _csv +import pandas as _pd +from collections import defaultdict as _defaultdict +from itertools import combinations as _c +import os as _os +import os.path as _ospath +import psutil as _psutil +import sys as _sys +import validators as _validators +import cupy as _cp +import numpy as _np +from urllib.request import urlopen as _urlopen + + +class _fuzzyFrequentPatterns(_ABC): + """ + :Description: This abstract base class defines the variables and methods that every fuzzy frequent pattern mining algorithm must + employ in PAMI + + :Attributes: + + iFile : str + Input file name or path of the input file + minSup: int or float or str + The user can specify minSup either in count or proportion of database size. + If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. + Otherwise, it will be treated as float. + Example: minSup=10 will be treated as integer, while minSup=10.0 will be treated as float + sep : str + This variable is used to distinguish items from one another in a transaction. The default seperator is tab space or \t. + However, the users can override their default separator. + startTime:float + To record the start time of the algorithm + endTime:float + To record the completion time of the algorithm + finalPatterns: dict + Storing the complete set of patterns in a dictionary variable + oFile : str + Name of the output file to store complete set of fuzzy frequent patterns + memoryUSS : float + To store the total amount of USS memory consumed by the program + memoryRSS : float + To store the total amount of RSS memory consumed by the program + + :Methods: + + mine() + Mining process will start from here + getPatterns() + Complete set of patterns will be retrieved with this function + save(oFile) + Complete set of fuzzy frequent patterns will be loaded in to a output file + getPatternsAsDataFrame() + Complete set of fuzzy frequent patterns will be loaded in to data frame + getMemoryUSS() + Total amount of USS memory consumed by the program will be retrieved from this function + getMemoryRSS() + Total amount of RSS memory consumed by the program will be retrieved from this function + getRuntime() + Total amount of runtime taken by the program will be retrieved from this function + """ + + def __init__(self, iFile, minSup, sep = '\t'): + """ + :param iFile: Input file name or path of the input file + :type iFile: str + :param minSup: The user can specify minSup either in count or proportion of database size. + If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. + Otherwise, it will be treated as float. + Example: minSup=10 will be treated as integer, while minSup=10.0 will be treated as float + :type minSup: int or float or str + :param sep: separator used in user specified input file + :type sep: str + """ + + self._iFile = iFile + self._minSup = minSup + self._sep = sep + self._finalPatterns = {} + self._startTime = float() + self._endTime = float() + self._memoryRSS = float() + self._memoryUSS = float() + self._oFile = " " + + @_abstractmethod + def startMine(self): + """ + Code for the mining process will start from this function + """ + + pass + + @_abstractmethod + def getPatterns(self): + """ + Complete set of fuzzy frequent patterns generated will be retrieved from this function + """ + + pass + + @_abstractmethod + def save(self, oFile): + """ + Complete set of fuzzy frequent patterns will be saved in to an output file from this function + :param oFile: Name of the output file + :type oFile: csvfile + """ + + pass + + @_abstractmethod + def getPatternsAsDataFrame(self): + """ + Complete set of fuzzy frequent patterns will be loaded in to data frame from this function + """ + + pass + + @_abstractmethod + def getMemoryUSS(self): + """ + Total amount of USS memory consumed by the program will be retrieved from this function + """ + + pass + + @_abstractmethod + def getMemoryRSS(self): + """ + Total amount of RSS memory consumed by the program will be retrieved from this function + """ + pass + + @_abstractmethod + def getRuntime(self): + """ + Total amount of runtime taken by the program will be retrieved from this function + """ + + pass + + @_abstractmethod + def printResults(self): + """ + To print results of the execution. + """ + + pass diff --git a/PAMI/fuzzyFrequentPattern/cuda/cuFFIMiner.py b/PAMI/fuzzyFrequentPattern/cuda/cuFFIMiner.py new file mode 100644 index 00000000..ca1cf576 --- /dev/null +++ b/PAMI/fuzzyFrequentPattern/cuda/cuFFIMiner.py @@ -0,0 +1,441 @@ +# cuFFIMiner is a fundamental algorithm to discover fuzzy frequent patterns in a quantitative transactional database using CUDA. This program employs the downward closure property to reduce the search space effectively. This algorithm employs breadth-first search technique to find the complete set of fuzzy frequent patterns in a quantitative transactional database. +# +# +# **Importing this algorithm into a python program** +# ---------------------------------------------------- +# +# import PAMI.fuzzyFrequentPattern.cuda.cuFFIMiner as alg +# +# obj = alg.cuFFIMiner(iFile, minSup) +# +# obj.mine() +# +# fuzzyFrequentPatterns = obj.getPatterns() +# +# print("Total number of Fuzzy Frequent Patterns:", len(fuzzyFrequentPatterns)) +# +# obj.save(oFile) +# +# Df = obj.getPatternInDataFrame() +# +# memUSS = obj.getMemoryUSS() +# +# print("Total Memory in USS:", memUSS) +# +# memRSS = obj.getMemoryRSS() +# +# print("Total Memory in RSS", memRSS) +# +# run = obj.getRuntime() +# +# print("Total ExecutionTime in seconds:", run) +# + + + + +__copyright__ = """ +Copyright (C) 2021 Rage Uday Kiran + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +from deprecated import deprecated +from PAMI.fuzzyFrequentPattern.cuda import abstract as _ab +# import abstract as _ab + +class cuFFIMiner(_ab._fuzzyFrequentPatterns): + """ + :Description: cuFFIMiner is a fundamental algorithm to discover fuzzy frequent patterns using Cuda in a quantitative transactional database. This program employs the downward closure property to reduce the search space effectively. This algorithm employs breadth-first search technique to find the complete set of fuzzy frequent patterns in a quantitative transactional database. + + :Reference: Lin, Chun-Wei & Li, Ting & Fournier Viger, Philippe & Hong, Tzung-Pei. (2015). + A fast Algorithm for mining fuzzy frequent itemsets. Journal of Intelligent & Fuzzy Systems. 29. + 2373-2379. 10.3233/IFS-151936. + https://www.researchgate.net/publication/286510908_A_fast_Algorithm_for_mining_fuzzy_frequent_itemSets + + :param iFile: str : + Name of the Input file to mine complete set of fuzzy frequent patterns + :param oFile: str : + Name of the output file to store complete set of fuzzy frequent patterns + :param minSup: int : + The user can specify minSup either in count or proportion of database size. If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. Otherwise, it will be treated as float. + :param sep: str : + This variable is used to distinguish items from one another in a transaction. The default seperator is tab space. However, the users can override their default separator. + + :Attributes: + + startTime : float + To record the start time of the mining process + + endTime : float + To record the completion time of the mining process + + finalPatterns : dict + Storing the complete set of patterns in a dictionary variable + + memoryUSS : float + To store the total amount of USS memory consumed by the program + + memoryRSS : float + To store the total amount of RSS memory consumed by the program + + Database : list + To store the transactions of a database in list + + + + **Methods to execute code on terminal** + ---------------------------------------------------- + + .. code-block:: console + + Format: + + (.venv) $ python3 cuFFIMiner.py + + Example Usage: + + (.venv) $ python3 cuFFIMiner.py sampleDB.txt patterns.txt 10.0 + + .. note:: minSup will be considered in percentage of database transactions + + + **Importing this algorithm into a python program** + ---------------------------------------------------- + + .. code-block:: python + + import PAMI.fuzzyFrequentPattern.cuda.cuFFIMiner as alg + + obj = alg.cuFFIMiner(iFile, minSup) + + obj.mine() + + fuzzyFrequentPatterns = obj.getPatterns() + + print("Total number of Fuzzy Frequent Patterns:", len(fuzzyFrequentPatterns)) + + obj.save(oFile) + + Df = obj.getPatternInDataFrame() + + memUSS = obj.getMemoryUSS() + + print("Total Memory in USS:", memUSS) + + memRSS = obj.getMemoryRSS() + + print("Total Memory in RSS", memRSS) + + run = obj.getRuntime() + + print("Total ExecutionTime in seconds:", run) + + + **Credits:** + ------------- + + The complete program was written by Mithun Thangaraj under the supervision of Professor Rage Uday Kiran. + + """ + + _minSup = float() + _startTime = float() + _endTime = float() + _finalPatterns = {} + _iFile = " " + _oFile = " " + _sep = "\t" + _memoryUSS = float() + _memoryRSS = float() + _transactions = [] + _fuzzyValues = [] + _dbLen = 0 + + _supportKernel = _ab._cp.RawKernel(r''' + + extern "C" __global__ + + void supportKernel(const float *matrix, const unsigned int *pairsA, const unsigned int *pairsB, + float *supports, unsigned int numElements, unsigned int numPairs) + { + __shared__ float partial[256]; + unsigned int p = blockIdx.x; + if (p >= numPairs) return; + const float *a = matrix + (unsigned long long)pairsA[p] * numElements; + const float *b = matrix + (unsigned long long)pairsB[p] * numElements; + float s = 0.0f; + for (unsigned int t = threadIdx.x; t < numElements; t += blockDim.x) + { + float va = a[t]; + float vb = b[t]; + s += va < vb ? va : vb; + } + partial[threadIdx.x] = s; + __syncthreads(); + for (unsigned int stride = blockDim.x / 2; stride > 0; stride >>= 1) + { + if (threadIdx.x < stride) + partial[threadIdx.x] += partial[threadIdx.x + stride]; + __syncthreads(); + } + if (threadIdx.x == 0) + supports[p] = partial[0]; + } + + ''', 'supportKernel') + + def _creatingItemSets(self): + """ + Storing the complete transactions of the database/input file in a database variable + """ + self._transactions, self._fuzzyValues = [], [] + if isinstance(self._iFile, _ab._pd.DataFrame): + if self._iFile.empty: + print("its empty..") + i = self._iFile.columns.values.tolist() + if 'Transactions' in i: + self._transactions = self._iFile['Transactions'].tolist() + if 'fuzzyValues' in i: + self._fuzzyValues = self._iFile['fuzzyValues'].tolist() + if isinstance(self._iFile, str): + if _ab._validators.url(self._iFile): + data = _ab._urlopen(self._iFile) + for line in data: + line = line.decode("utf-8") + line = line.split("\n")[0] + parts = line.split(":") + parts[0] = parts[0].strip() + parts[1] = parts[1].strip() + items = parts[0].split(self._sep) + quantities = parts[1].split(self._sep) + self._transactions.append([x for x in items if x]) + self._fuzzyValues.append([float(x) for x in quantities if x]) + else: + try: + with open(self._iFile, 'r', encoding='utf-8') as f: + for line in f: + line = line.split("\n")[0] + parts = line.split(":") + parts[0] = parts[0].strip() + parts[1] = parts[1].strip() + items = parts[0].split(self._sep) + quantities = parts[1].split(self._sep) + self._transactions.append([x for x in items if x]) + self._fuzzyValues.append([float(x) for x in quantities if x]) + except IOError: + print("File Not Found") + quit() + + def _convert(self, value): + """ + + To convert the user specified minSup value + + :param value: user specified minSup value + + :type value: int or float or str + + :return: converted type + + """ + if type(value) is int: + value = int(value) + if type(value) is float: + value = (self._dbLen * value) + if type(value) is str: + if '.' in value: + value = float(value) + value = (self._dbLen * value) + else: + value = int(value) + return value + + def arraysAndItems(self): + ArraysAndItems = {} + + for i in range(self._dbLen): + for item, fuzzyValue in zip(self._transactions[i], self._fuzzyValues[i]): + j = tuple([item]) + if j not in ArraysAndItems: + ArraysAndItems[j] = _ab._np.zeros(self._dbLen, dtype=_ab._np.float32) + ArraysAndItems[j][i] = fuzzyValue + + newArraysAndItems = {} + + for k, v in ArraysAndItems.items(): + support = float(v.sum()) + if support >= self._minSup: + self._finalPatterns[k] = support + newArraysAndItems[k] = _ab._cp.array(v) + + return newArraysAndItems + + @deprecated("It is recommended to use 'mine()' instead of 'startMine()' for mining process. Starting from January 2025, 'startMine()' will be completely terminated.") + def startMine(self): + """ + Fuzzy frequent pattern mining process will start from here + """ + self.mine() + + def mine(self): + """ + Fuzzy frequent pattern mining process will start from here + """ + _ab._cp.cuda.Device(0).use() + self._startTime = _ab._time.time() + self._creatingItemSets() + self._dbLen = len(self._transactions) + self._minSup = self._convert(self._minSup) + + ArraysAndItems = self.arraysAndItems() + + while len(ArraysAndItems) > 0: + # print("Total number of ArraysAndItems:", len(ArraysAndItems)) + newArraysAndItems = {} + keys = list(ArraysAndItems.keys()) + + pairsA, pairsB, unions = [], [], [] + seen = set() + for i in range(len(ArraysAndItems)): + # print(i, "/", len(ArraysAndItems), end="\r") + iList = list(keys[i]) + for j in range(i + 1, len(ArraysAndItems)): + jList = list(keys[j]) + union = tuple(sorted(set(iList + jList))) + if union in self._finalPatterns or union in seen: + continue + seen.add(union) + pairsA.append(i) + pairsB.append(j) + unions.append(union) + + if len(pairsA) > 0: + numPairs = len(pairsA) + matrix = _ab._cp.stack(list(ArraysAndItems.values())) + pairsA = _ab._np.asarray(pairsA, dtype=_ab._np.uint32) + pairsB = _ab._np.asarray(pairsB, dtype=_ab._np.uint32) + supports = _ab._cp.zeros(numPairs, dtype=_ab._cp.float32) + self._supportKernel((numPairs,), (256,), + (matrix, _ab._cp.array(pairsA), _ab._cp.array(pairsB), supports, + _ab._np.uint32(self._dbLen), _ab._np.uint32(numPairs))) + supports = supports.get() + + survivors = _ab._np.where(supports >= self._minSup)[0] + if len(survivors) > 0: + survA = _ab._cp.array(pairsA[survivors]) + survB = _ab._cp.array(pairsB[survivors]) + newMatrix = _ab._cp.minimum(matrix[survA], matrix[survB]) + for row, idx in enumerate(survivors): + union = unions[idx] + self._finalPatterns[union] = float(supports[idx]) + newArraysAndItems[union] = newMatrix[row] + + ArraysAndItems = newArraysAndItems + # print() + + self._endTime = _ab._time.time() + process = _ab._psutil.Process(_ab._os.getpid()) + self._memoryUSS = float() + self._memoryRSS = float() + self._memoryUSS = process.memory_full_info().uss + self._memoryRSS = process.memory_info().rss + print("Fuzzy frequent patterns were generated successfully using cuFFIMiner algorithm ") + + def getMemoryUSS(self): + """ + Total amount of USS memory consumed by the mining process will be retrieved from this function + :return: returning USS memory consumed by the mining process + :rtype: float + """ + + return self._memoryUSS + + def getMemoryRSS(self): + """ + Total amount of RSS memory consumed by the mining process will be retrieved from this function + :return: returning RSS memory consumed by the mining process + :rtype: float + """ + + return self._memoryRSS + + def getRuntime(self): + """ + Calculating the total amount of runtime taken by the mining process + :return: returning total amount of runtime taken by the mining process + :rtype: float + """ + + return self._endTime - self._startTime + + def getPatternsAsDataFrame(self): + """ + Storing final fuzzy frequent patterns in a dataframe + :return: returning fuzzy frequent patterns in a dataframe + :rtype: pd.DataFrame + """ + + dataFrame = {} + data = [] + for a, b in self._finalPatterns.items(): + data.append([" ".join(a), b]) + dataFrame = _ab._pd.DataFrame(data, columns=['Patterns', 'Support']) + # dataFrame = dataFrame.replace(r'\r+|\n+|\t+',' ', regex=True) + return dataFrame + + def save(self, outFile): + """ + Complete set of fuzzy frequent patterns will be loaded in to an output file + :param outFile: name of the output file + :type outFile: csvfile + """ + self._oFile = outFile + writer = open(self._oFile, 'w+') + for x, y in self._finalPatterns.items(): + s1 = "\t".join(x) + ":" + str(y) + writer.write("%s \n" % s1) + + def getPatterns(self): + """ + Function to send the set of fuzzy frequent patterns after completion of the mining process + :return: returning fuzzy frequent patterns + :rtype: dict + """ + return self._finalPatterns + + def printResults(self): + """ + This function is used to print results + """ + print("Total number of Fuzzy Frequent Patterns:", len(self.getPatterns())) + print("Total Memory in USS:", self.getMemoryUSS()) + print("Total Memory in RSS", self.getMemoryRSS()) + print("Total ExecutionTime in s:", self.getRuntime()) + +if __name__ == "__main__": + _ap = str() + if len(_ab._sys.argv) == 4 or len(_ab._sys.argv) == 5: + if len(_ab._sys.argv) == 5: + _ap = cuFFIMiner(_ab._sys.argv[1], _ab._sys.argv[3], _ab._sys.argv[4]) + if len(_ab._sys.argv) == 4: + _ap = cuFFIMiner(_ab._sys.argv[1], _ab._sys.argv[3]) + _ap.mine() + print("Total number of Fuzzy Frequent Patterns:", len(_ap.getPatterns())) + _ap.save(_ab._sys.argv[2]) + print("Total Memory in USS:", _ap.getMemoryUSS()) + print("Total Memory in RSS", _ap.getMemoryRSS()) + print("Total ExecutionTime in s:", _ap.getRuntime()) + else: + print("Error! The number of input parameters do not match the total number of parameters provided") diff --git a/PAMI/fuzzyPartialPeriodicPatterns/cuda/__init__.py b/PAMI/fuzzyPartialPeriodicPatterns/cuda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/PAMI/fuzzyPartialPeriodicPatterns/cuda/abstract.py b/PAMI/fuzzyPartialPeriodicPatterns/cuda/abstract.py new file mode 100644 index 00000000..2bb2862f --- /dev/null +++ b/PAMI/fuzzyPartialPeriodicPatterns/cuda/abstract.py @@ -0,0 +1,167 @@ +# Copyright (C) 2021 Rage Uday Kiran +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from abc import ABC as _ABC, abstractmethod as _abstractmethod +import time as _time +import math as _math +import csv as _csv +import pandas as _pd +from collections import defaultdict as _defaultdict +from itertools import combinations as _c +import os as _os +import os.path as _ospath +import psutil as _psutil +import sys as _sys +import validators as _validators +import cupy as _cp +import numpy as _np +from urllib.request import urlopen as _urlopen + + +class _fuzzyPartialPeriodicPatterns(_ABC): + """ + :Description: This abstract base class defines the variables and methods that every fuzzy partial periodic pattern mining algorithm must + employ in PAMI + + :Attributes: + + iFile : str + Input file name or path of the input file + minSup: int or float or str + The user can specify minSup either in count or proportion of database size. + If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. + Otherwise, it will be treated as float. + Example: minSup=10 will be treated as integer, while minSup=10.0 will be treated as float + sep : str + This variable is used to distinguish items from one another in a transaction. The default seperator is tab space or \t. + However, the users can override their default separator. + startTime:float + To record the start time of the algorithm + endTime:float + To record the completion time of the algorithm + finalPatterns: dict + Storing the complete set of patterns in a dictionary variable + oFile : str + Name of the output file to store complete set of fuzzy partial periodic patterns + memoryUSS : float + To store the total amount of USS memory consumed by the program + memoryRSS : float + To store the total amount of RSS memory consumed by the program + + :Methods: + + mine() + Mining process will start from here + getPatterns() + Complete set of patterns will be retrieved with this function + save(oFile) + Complete set of fuzzy partial periodic patterns will be loaded in to a output file + getPatternsAsDataFrame() + Complete set of fuzzy partial periodic patterns will be loaded in to data frame + getMemoryUSS() + Total amount of USS memory consumed by the program will be retrieved from this function + getMemoryRSS() + Total amount of RSS memory consumed by the program will be retrieved from this function + getRuntime() + Total amount of runtime taken by the program will be retrieved from this function + """ + + def __init__(self, iFile, minSup, sep = '\t'): + """ + :param iFile: Input file name or path of the input file + :type iFile: str + :param minSup: The user can specify minSup either in count or proportion of database size. + If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. + Otherwise, it will be treated as float. + Example: minSup=10 will be treated as integer, while minSup=10.0 will be treated as float + :type minSup: int or float or str + :param sep: separator used in user specified input file + :type sep: str + """ + + self._iFile = iFile + self._minSup = minSup + self._sep = sep + self._finalPatterns = {} + self._startTime = float() + self._endTime = float() + self._memoryRSS = float() + self._memoryUSS = float() + self._oFile = " " + + @_abstractmethod + def startMine(self): + """ + Code for the mining process will start from this function + """ + + pass + + @_abstractmethod + def getPatterns(self): + """ + Complete set of fuzzy partial periodic patterns generated will be retrieved from this function + """ + + pass + + @_abstractmethod + def save(self, oFile): + """ + Complete set of fuzzy partial periodic patterns will be saved in to an output file from this function + :param oFile: Name of the output file + :type oFile: csvfile + """ + + pass + + @_abstractmethod + def getPatternsAsDataFrame(self): + """ + Complete set of fuzzy partial periodic patterns will be loaded in to data frame from this function + """ + + pass + + @_abstractmethod + def getMemoryUSS(self): + """ + Total amount of USS memory consumed by the program will be retrieved from this function + """ + + pass + + @_abstractmethod + def getMemoryRSS(self): + """ + Total amount of RSS memory consumed by the program will be retrieved from this function + """ + pass + + @_abstractmethod + def getRuntime(self): + """ + Total amount of runtime taken by the program will be retrieved from this function + """ + + pass + + @_abstractmethod + def printResults(self): + """ + To print results of the execution. + """ + + pass diff --git a/PAMI/fuzzyPartialPeriodicPatterns/cuda/cuF3PMiner.py b/PAMI/fuzzyPartialPeriodicPatterns/cuda/cuF3PMiner.py new file mode 100644 index 00000000..96367e31 --- /dev/null +++ b/PAMI/fuzzyPartialPeriodicPatterns/cuda/cuF3PMiner.py @@ -0,0 +1,439 @@ +# cuF3PMiner algorithm discovers the fuzzy partial periodic patterns in quantitative irregular multiple timeseries databases using CUDA. This program employs the downward closure property to reduce the search space effectively. This algorithm employs breadth-first search technique to find the complete set of fuzzy partial periodic patterns. +# +# +# **Importing this algorithm into a python program** +# ---------------------------------------------------- +# +# import PAMI.fuzzyPartialPeriodicPatterns.cuda.cuF3PMiner as alg +# +# obj = alg.cuF3PMiner(iFile, minSup) +# +# obj.mine() +# +# fuzzyPartialPeriodicPatterns = obj.getPatterns() +# +# print("Total number of Fuzzy Partial Periodic Patterns:", len(fuzzyPartialPeriodicPatterns)) +# +# obj.save(oFile) +# +# Df = obj.getPatternInDataFrame() +# +# memUSS = obj.getMemoryUSS() +# +# print("Total Memory in USS:", memUSS) +# +# memRSS = obj.getMemoryRSS() +# +# print("Total Memory in RSS", memRSS) +# +# run = obj.getRuntime() +# +# print("Total ExecutionTime in seconds:", run) +# + + + + +__copyright__ = """ +Copyright (C) 2021 Rage Uday Kiran + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +from deprecated import deprecated +from PAMI.fuzzyPartialPeriodicPatterns.cuda import abstract as _ab +# import abstract as _ab + +class cuF3PMiner(_ab._fuzzyPartialPeriodicPatterns): + """ + :Description: cuF3PMiner algorithm discovers the fuzzy partial periodic patterns in quantitative irregular multiple timeseries databases using Cuda. This program employs the downward closure property to reduce the search space effectively. This algorithm employs breadth-first search technique to find the complete set of fuzzy partial periodic patterns. + + :param iFile: str : + Name of the Input file to mine complete set of fuzzy partial periodic patterns + :param oFile: str : + Name of the output file to store complete set of fuzzy partial periodic patterns + :param minSup: int : + The user can specify minSup either in count or proportion of database size. If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. Otherwise, it will be treated as float. + :param sep: str : + This variable is used to distinguish items from one another in a transaction. The default seperator is tab space. However, the users can override their default separator. + + :Attributes: + + startTime : float + To record the start time of the mining process + + endTime : float + To record the completion time of the mining process + + finalPatterns : dict + Storing the complete set of patterns in a dictionary variable + + memoryUSS : float + To store the total amount of USS memory consumed by the program + + memoryRSS : float + To store the total amount of RSS memory consumed by the program + + Database : list + To store the transactions of a database in list + + + + **Methods to execute code on terminal** + ---------------------------------------------------- + + .. code-block:: console + + Format: + + (.venv) $ python3 cuF3PMiner.py + + Example Usage: + + (.venv) $ python3 cuF3PMiner.py sampleDB.txt patterns.txt 10.0 + + .. note:: minSup will be considered in percentage of database transactions + + + **Importing this algorithm into a python program** + ---------------------------------------------------- + + .. code-block:: python + + import PAMI.fuzzyPartialPeriodicPatterns.cuda.cuF3PMiner as alg + + obj = alg.cuF3PMiner(iFile, minSup) + + obj.mine() + + fuzzyPartialPeriodicPatterns = obj.getPatterns() + + print("Total number of Fuzzy Partial Periodic Patterns:", len(fuzzyPartialPeriodicPatterns)) + + obj.save(oFile) + + Df = obj.getPatternInDataFrame() + + memUSS = obj.getMemoryUSS() + + print("Total Memory in USS:", memUSS) + + memRSS = obj.getMemoryRSS() + + print("Total Memory in RSS", memRSS) + + run = obj.getRuntime() + + print("Total ExecutionTime in seconds:", run) + + + **Credits:** + ------------- + + The complete program was written by Mithun Thangaraj under the supervision of Professor Rage Uday Kiran. + + """ + + _minSup = float() + _startTime = float() + _endTime = float() + _finalPatterns = {} + _iFile = " " + _oFile = " " + _sep = "\t" + _memoryUSS = float() + _memoryRSS = float() + _transactions = [] + _fuzzyValues = [] + _ts = [] + _dbLen = 0 + + _supportKernel = _ab._cp.RawKernel(r''' + + extern "C" __global__ + + void supportKernel(const float *matrix, const unsigned int *pairsA, const unsigned int *pairsB, + float *supports, unsigned int numElements, unsigned int numPairs) + { + __shared__ float partial[256]; + unsigned int p = blockIdx.x; + if (p >= numPairs) return; + const float *a = matrix + (unsigned long long)pairsA[p] * numElements; + const float *b = matrix + (unsigned long long)pairsB[p] * numElements; + float s = 0.0f; + for (unsigned int t = threadIdx.x; t < numElements; t += blockDim.x) + { + float va = a[t]; + float vb = b[t]; + s += va < vb ? va : vb; + } + partial[threadIdx.x] = s; + __syncthreads(); + for (unsigned int stride = blockDim.x / 2; stride > 0; stride >>= 1) + { + if (threadIdx.x < stride) + partial[threadIdx.x] += partial[threadIdx.x + stride]; + __syncthreads(); + } + if (threadIdx.x == 0) + supports[p] = partial[0]; + } + + ''', 'supportKernel') + + def _creatingItemSets(self): + """ + Storing the complete transactions of the database/input file in a database variable + """ + self._transactions, self._fuzzyValues, self._ts = [], [], [] + if isinstance(self._iFile, _ab._pd.DataFrame): + if self._iFile.empty: + print("its empty..") + i = self._iFile.columns.values.tolist() + if 'Transactions' in i: + self._transactions = self._iFile['Transactions'].tolist() + if 'fuzzyValues' in i: + self._fuzzyValues = self._iFile['fuzzyValues'].tolist() + if isinstance(self._iFile, str): + if _ab._validators.url(self._iFile): + data = _ab._urlopen(self._iFile) + for line in data: + line = line.decode("utf-8") + line = line.strip() + parts = line.split(":") + times = [x for x in parts[0].strip().split(self._sep) if x] + items = [x for x in parts[1].strip().split(self._sep) if x] + quantities = [float(x) for x in parts[2].strip().split(self._sep) if x] + tempList = ["(" + times[k] + "," + items[k] + ")" for k in range(len(times))] + self._ts.append(times) + self._transactions.append(tempList) + self._fuzzyValues.append(quantities) + else: + try: + with open(self._iFile, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + parts = line.split(":") + times = [x for x in parts[0].strip().split(self._sep) if x] + items = [x for x in parts[1].strip().split(self._sep) if x] + quantities = [float(x) for x in parts[2].strip().split(self._sep) if x] + tempList = ["(" + times[k] + "," + items[k] + ")" for k in range(len(times))] + self._ts.append(times) + self._transactions.append(tempList) + self._fuzzyValues.append(quantities) + except IOError: + print("File Not Found") + quit() + + def _convert(self, value): + """ + + To convert the user specified minSup value + + :param value: user specified minSup value + + :type value: int or float or str + + :return: converted type + + """ + if type(value) is int: + value = int(value) + if type(value) is float: + value = (self._dbLen * value) + if type(value) is str: + if '.' in value: + value = float(value) + value = (self._dbLen * value) + else: + value = int(value) + return value + + def arraysAndItems(self): + ArraysAndItems = {} + + for i in range(self._dbLen): + for item, fuzzyValue in zip(self._transactions[i], self._fuzzyValues[i]): + j = tuple([item]) + if j not in ArraysAndItems: + ArraysAndItems[j] = _ab._np.zeros(self._dbLen, dtype=_ab._np.float32) + ArraysAndItems[j][i] += fuzzyValue + + newArraysAndItems = {} + + for k, v in ArraysAndItems.items(): + support = float(v.sum()) + if support >= self._minSup: + self._finalPatterns[k] = support + newArraysAndItems[k] = _ab._cp.array(v) + + return newArraysAndItems + + @deprecated("It is recommended to use 'mine()' instead of 'startMine()' for mining process. Starting from January 2025, 'startMine()' will be completely terminated.") + def startMine(self): + """ + Fuzzy partial periodic pattern mining process will start from here + """ + self.mine() + + def mine(self): + """ + Fuzzy partial periodic pattern mining process will start from here + """ + _ab._cp.cuda.Device(0).use() + self._startTime = _ab._time.time() + self._creatingItemSets() + self._dbLen = len(self._transactions) + self._minSup = self._convert(self._minSup) + + ArraysAndItems = self.arraysAndItems() + + while len(ArraysAndItems) > 0: + # print("Total number of ArraysAndItems:", len(ArraysAndItems)) + newArraysAndItems = {} + keys = list(ArraysAndItems.keys()) + + pairsA, pairsB, unions = [], [], [] + seen = set() + for i in range(len(ArraysAndItems)): + # print(i, "/", len(ArraysAndItems), end="\r") + iList = list(keys[i]) + for j in range(i + 1, len(ArraysAndItems)): + jList = list(keys[j]) + union = tuple(sorted(set(iList + jList))) + if union in self._finalPatterns or union in seen: + continue + seen.add(union) + pairsA.append(i) + pairsB.append(j) + unions.append(union) + + if len(pairsA) > 0: + numPairs = len(pairsA) + matrix = _ab._cp.stack(list(ArraysAndItems.values())) + pairsA = _ab._np.asarray(pairsA, dtype=_ab._np.uint32) + pairsB = _ab._np.asarray(pairsB, dtype=_ab._np.uint32) + supports = _ab._cp.zeros(numPairs, dtype=_ab._cp.float32) + self._supportKernel((numPairs,), (256,), + (matrix, _ab._cp.array(pairsA), _ab._cp.array(pairsB), supports, + _ab._np.uint32(self._dbLen), _ab._np.uint32(numPairs))) + supports = supports.get() + + survivors = _ab._np.where(supports >= self._minSup)[0] + if len(survivors) > 0: + survA = _ab._cp.array(pairsA[survivors]) + survB = _ab._cp.array(pairsB[survivors]) + newMatrix = _ab._cp.minimum(matrix[survA], matrix[survB]) + for row, idx in enumerate(survivors): + union = unions[idx] + self._finalPatterns[union] = float(supports[idx]) + newArraysAndItems[union] = newMatrix[row] + + ArraysAndItems = newArraysAndItems + # print() + + self._endTime = _ab._time.time() + process = _ab._psutil.Process(_ab._os.getpid()) + self._memoryUSS = float() + self._memoryRSS = float() + self._memoryUSS = process.memory_full_info().uss + self._memoryRSS = process.memory_info().rss + print("Fuzzy partial periodic patterns were generated successfully using cuF3PMiner algorithm ") + + def getMemoryUSS(self): + """ + Total amount of USS memory consumed by the mining process will be retrieved from this function + :return: returning USS memory consumed by the mining process + :rtype: float + """ + + return self._memoryUSS + + def getMemoryRSS(self): + """ + Total amount of RSS memory consumed by the mining process will be retrieved from this function + :return: returning RSS memory consumed by the mining process + :rtype: float + """ + + return self._memoryRSS + + def getRuntime(self): + """ + Calculating the total amount of runtime taken by the mining process + :return: returning total amount of runtime taken by the mining process + :rtype: float + """ + + return self._endTime - self._startTime + + def getPatternsAsDataFrame(self): + """ + Storing final fuzzy partial periodic patterns in a dataframe + :return: returning fuzzy partial periodic patterns in a dataframe + :rtype: pd.DataFrame + """ + + dataFrame = {} + data = [] + for a, b in self._finalPatterns.items(): + data.append([" ".join(a), b]) + dataFrame = _ab._pd.DataFrame(data, columns=['Patterns', 'Support']) + # dataFrame = dataFrame.replace(r'\r+|\n+|\t+',' ', regex=True) + return dataFrame + + def save(self, outFile): + """ + Complete set of fuzzy partial periodic patterns will be loaded in to an output file + :param outFile: name of the output file + :type outFile: csvfile + """ + self._oFile = outFile + writer = open(self._oFile, 'w+') + for x, y in self._finalPatterns.items(): + s1 = "\t".join(x) + ":" + str(y) + writer.write("%s \n" % s1) + + def getPatterns(self): + """ + Function to send the set of fuzzy partial periodic patterns after completion of the mining process + :return: returning fuzzy partial periodic patterns + :rtype: dict + """ + return self._finalPatterns + + def printResults(self): + """ + This function is used to print results + """ + print("Total number of Fuzzy Partial Periodic Patterns:", len(self.getPatterns())) + print("Total Memory in USS:", self.getMemoryUSS()) + print("Total Memory in RSS", self.getMemoryRSS()) + print("Total ExecutionTime in s:", self.getRuntime()) + +if __name__ == "__main__": + _ap = str() + if len(_ab._sys.argv) == 4 or len(_ab._sys.argv) == 5: + if len(_ab._sys.argv) == 5: + _ap = cuF3PMiner(_ab._sys.argv[1], _ab._sys.argv[3], _ab._sys.argv[4]) + if len(_ab._sys.argv) == 4: + _ap = cuF3PMiner(_ab._sys.argv[1], _ab._sys.argv[3]) + _ap.mine() + print("Total number of Fuzzy Partial Periodic Patterns:", len(_ap.getPatterns())) + _ap.save(_ab._sys.argv[2]) + print("Total Memory in USS:", _ap.getMemoryUSS()) + print("Total Memory in RSS", _ap.getMemoryRSS()) + print("Total ExecutionTime in s:", _ap.getRuntime()) + else: + print("Error! The number of input parameters do not match the total number of parameters provided") diff --git a/PAMI/fuzzyPeriodicFrequentPattern/cuda/__init__.py b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/__init__.cpython-312.pyc b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 00000000..b810485f Binary files /dev/null and b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/__init__.cpython-312.pyc differ diff --git a/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/abstract.cpython-312.pyc b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/abstract.cpython-312.pyc new file mode 100644 index 00000000..8afdf388 Binary files /dev/null and b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/abstract.cpython-312.pyc differ diff --git a/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/cuFPFPMiner.cpython-312.pyc b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/cuFPFPMiner.cpython-312.pyc new file mode 100644 index 00000000..aa0cec73 Binary files /dev/null and b/PAMI/fuzzyPeriodicFrequentPattern/cuda/__pycache__/cuFPFPMiner.cpython-312.pyc differ diff --git a/PAMI/fuzzyPeriodicFrequentPattern/cuda/abstract.py b/PAMI/fuzzyPeriodicFrequentPattern/cuda/abstract.py new file mode 100644 index 00000000..cc89006f --- /dev/null +++ b/PAMI/fuzzyPeriodicFrequentPattern/cuda/abstract.py @@ -0,0 +1,174 @@ +# Copyright (C) 2021 Rage Uday Kiran +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from abc import ABC as _ABC, abstractmethod as _abstractmethod +import time as _time +import math as _math +import csv as _csv +import pandas as _pd +from collections import defaultdict as _defaultdict +from itertools import combinations as _c +import os as _os +import os.path as _ospath +import psutil as _psutil +import sys as _sys +import validators as _validators +import cupy as _cp +import numpy as _np +from urllib.request import urlopen as _urlopen + + +class _fuzzyPeriodicFrequentPatterns(_ABC): + """ + :Description: This abstract base class defines the variables and methods that every fuzzy periodic frequent + pattern mining algorithm must employ in PAMI + + :Attributes: + + iFile : str + Input file name or path of the input file + minSup: int or float or str + The user can specify minSup either in count or proportion of database size. + If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. + Otherwise, it will be treated as float. + Example: minSup=10 will be treated as integer, while minSup=10.0 will be treated as float + maxPer: int or float + The user can specify maxPer either in count or proportion of database size. + If the program detects the data type of maxPer is integer, then it treats maxPer is expressed in count. + Otherwise, it will be treated as float. + sep : str + This variable is used to distinguish items from one another in a transaction. The default seperator is tab space or \t. + However, the users can override their default separator. + startTime:float + To record the start time of the algorithm + endTime:float + To record the completion time of the algorithm + finalPatterns: dict + Storing the complete set of patterns in a dictionary variable + oFile : str + Name of the output file to store complete set of fuzzy periodic frequent patterns + memoryUSS : float + To store the total amount of USS memory consumed by the program + memoryRSS : float + To store the total amount of RSS memory consumed by the program + + :Methods: + + mine() + Mining process will start from here + getPatterns() + Complete set of patterns will be retrieved with this function + save(oFile) + Complete set of fuzzy periodic frequent patterns will be loaded in to a output file + getPatternsAsDataFrame() + Complete set of fuzzy periodic frequent patterns will be loaded in to data frame + getMemoryUSS() + Total amount of USS memory consumed by the program will be retrieved from this function + getMemoryRSS() + Total amount of RSS memory consumed by the program will be retrieved from this function + getRuntime() + Total amount of runtime taken by the program will be retrieved from this function + """ + + def __init__(self, iFile, minSup, maxPer, sep = '\t'): + """ + :param iFile: Input file name or path of the input file + :type iFile: str + :param minSup: The user can specify minSup either in count or proportion of database size. + If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. + Otherwise, it will be treated as float. + Example: minSup=10 will be treated as integer, while minSup=10.0 will be treated as float + :type minSup: int or float or str + :param maxPer: The user can specify maximum periodicity either in count or proportion of database size. + :type maxPer: int or float + :param sep: separator used in user specified input file + :type sep: str + """ + + self._iFile = iFile + self._minSup = minSup + self._maxPer = maxPer + self._sep = sep + self._finalPatterns = {} + self._startTime = float() + self._endTime = float() + self._memoryRSS = float() + self._memoryUSS = float() + self._oFile = " " + + @_abstractmethod + def startMine(self): + """ + Code for the mining process will start from this function + """ + + pass + + @_abstractmethod + def getPatterns(self): + """ + Complete set of fuzzy periodic frequent patterns generated will be retrieved from this function + """ + + pass + + @_abstractmethod + def save(self, oFile): + """ + Complete set of fuzzy periodic frequent patterns will be saved in to an output file from this function + :param oFile: Name of the output file + :type oFile: csvfile + """ + + pass + + @_abstractmethod + def getPatternsAsDataFrame(self): + """ + Complete set of fuzzy periodic frequent patterns will be loaded in to data frame from this function + """ + + pass + + @_abstractmethod + def getMemoryUSS(self): + """ + Total amount of USS memory consumed by the program will be retrieved from this function + """ + + pass + + @_abstractmethod + def getMemoryRSS(self): + """ + Total amount of RSS memory consumed by the program will be retrieved from this function + """ + pass + + @_abstractmethod + def getRuntime(self): + """ + Total amount of runtime taken by the program will be retrieved from this function + """ + + pass + + @_abstractmethod + def printResults(self): + """ + To print results of the execution. + """ + + pass diff --git a/PAMI/fuzzyPeriodicFrequentPattern/cuda/cuFPFPMiner.py b/PAMI/fuzzyPeriodicFrequentPattern/cuda/cuFPFPMiner.py new file mode 100644 index 00000000..43ef7ab9 --- /dev/null +++ b/PAMI/fuzzyPeriodicFrequentPattern/cuda/cuFPFPMiner.py @@ -0,0 +1,505 @@ +# cuFPFPMiner is a fundamental algorithm to discover fuzzy periodic frequent patterns in a quantitative temporal database using CUDA. This program employs the downward closure property to reduce the search space effectively. This algorithm employs breadth-first search technique to find the complete set of fuzzy periodic frequent patterns in a quantitative temporal database. +# +# +# **Importing this algorithm into a python program** +# ---------------------------------------------------- +# +# import PAMI.fuzzyPeriodicFrequentPattern.cuda.cuFPFPMiner as alg +# +# obj = alg.cuFPFPMiner(iFile, minSup, maxPer) +# +# obj.mine() +# +# fuzzyPeriodicFrequentPatterns = obj.getPatterns() +# +# print("Total number of Fuzzy Periodic Frequent Patterns:", len(fuzzyPeriodicFrequentPatterns)) +# +# obj.save(oFile) +# +# Df = obj.getPatternsAsDataFrame() +# +# memUSS = obj.getMemoryUSS() +# +# print("Total Memory in USS:", memUSS) +# +# memRSS = obj.getMemoryRSS() +# +# print("Total Memory in RSS", memRSS) +# +# run = obj.getRuntime() +# +# print("Total ExecutionTime in seconds:", run) +# + + + + +__copyright__ = """ +Copyright (C) 2021 Rage Uday Kiran + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +from deprecated import deprecated +from PAMI.fuzzyPeriodicFrequentPattern.cuda import abstract as _ab +# import abstract as _ab + +class cuFPFPMiner(_ab._fuzzyPeriodicFrequentPatterns): + """ + :Description: cuFPFPMiner is a fundamental algorithm to discover fuzzy periodic frequent patterns using Cuda in a quantitative temporal database. This program employs the downward closure property to reduce the search space effectively. This algorithm employs breadth-first search technique to find the complete set of fuzzy periodic frequent patterns in a quantitative temporal database. + + :Reference: R. U. Kiran et al., "Discovering Fuzzy Periodic-Frequent Patterns in Quantitative Temporal Databases," + 2020 IEEE International Conference on Fuzzy Systems (FUZZ-IEEE), Glasgow, UK, 2020, pp. + 1-8, doi: 10.1109/FUZZ48607.2020.9177579. + + :param iFile: str : + Name of the Input file to mine complete set of fuzzy periodic frequent patterns + :param oFile: str : + Name of the output file to store complete set of fuzzy periodic frequent patterns + :param minSup: int or float or str : + The user can specify minSup either in count or proportion of database size. If the program detects the data type of minSup is integer, then it treats minSup is expressed in count. Otherwise, it will be treated as float. + :param maxPer: int or float : + The user can specify maxPer either in count or proportion of database size. If the program detects the data type of maxPer is integer, then it treats maxPer is expressed in count. + :param sep: str : + This variable is used to distinguish items from one another in a transaction. The default seperator is tab space. However, the users can override their default separator. + + :Attributes: + + startTime : float + To record the start time of the mining process + + endTime : float + To record the completion time of the mining process + + finalPatterns : dict + Storing the complete set of patterns in a dictionary variable + + memoryUSS : float + To store the total amount of USS memory consumed by the program + + memoryRSS : float + To store the total amount of RSS memory consumed by the program + + Database : list + To store the transactions of a database in list + + + + **Methods to execute code on terminal** + ---------------------------------------------------- + + .. code-block:: console + + Format: + + (.venv) $ python3 cuFPFPMiner.py + + Example Usage: + + (.venv) $ python3 cuFPFPMiner.py sampleDB.txt patterns.txt 10.0 3 + + .. note:: minSup will be considered in percentage of database transactions + + + **Importing this algorithm into a python program** + ---------------------------------------------------- + + .. code-block:: python + + import PAMI.fuzzyPeriodicFrequentPattern.cuda.cuFPFPMiner as alg + + obj = alg.cuFPFPMiner(iFile, minSup, maxPer) + + obj.mine() + + fuzzyPeriodicFrequentPatterns = obj.getPatterns() + + print("Total number of Fuzzy Periodic Frequent Patterns:", len(fuzzyPeriodicFrequentPatterns)) + + obj.save(oFile) + + Df = obj.getPatternsAsDataFrame() + + memUSS = obj.getMemoryUSS() + + print("Total Memory in USS:", memUSS) + + memRSS = obj.getMemoryRSS() + + print("Total Memory in RSS", memRSS) + + run = obj.getRuntime() + + print("Total ExecutionTime in seconds:", run) + + + **Credits:** + ------------- + + The complete program was written by Mithun Thangaraj under the supervision of Professor Rage Uday Kiran. + + """ + + _minSup = float() + _maxPer = float() + _startTime = float() + _endTime = float() + _finalPatterns = {} + _iFile = " " + _oFile = " " + _sep = "\t" + _memoryUSS = float() + _memoryRSS = float() + _transactions = [] + _fuzzyValues = [] + _ts = [] + _dbLen = 0 + + _supportKernel = _ab._cp.RawKernel(r''' + + extern "C" __global__ + + void supportKernel(const float *matrix, const unsigned int *pairsA, const unsigned int *pairsB, + float *supports, unsigned int numElements, unsigned int numPairs) + { + __shared__ float partial[256]; + unsigned int p = blockIdx.x; + if (p >= numPairs) return; + const float *a = matrix + (unsigned long long)pairsA[p] * numElements; + const float *b = matrix + (unsigned long long)pairsB[p] * numElements; + float s = 0.0f; + for (unsigned int t = threadIdx.x; t < numElements; t += blockDim.x) + { + float va = a[t]; + float vb = b[t]; + s += va < vb ? va : vb; + } + partial[threadIdx.x] = s; + __syncthreads(); + for (unsigned int stride = blockDim.x / 2; stride > 0; stride >>= 1) + { + if (threadIdx.x < stride) + partial[threadIdx.x] += partial[threadIdx.x + stride]; + __syncthreads(); + } + if (threadIdx.x == 0) + supports[p] = partial[0]; + } + + ''', 'supportKernel') + + _periodKernel = _ab._cp.RawKernel(r''' + + extern "C" __global__ + + void periodKernel(const float *matrix, const float *ts, float *maxPeriods, + unsigned int numElements, unsigned int numRows) + { + unsigned int r = blockIdx.x * blockDim.x + threadIdx.x; + if (r >= numRows) return; + const float *row = matrix + (unsigned long long)r * numElements; + float prev = 0.0f; + float maxGap = 0.0f; + for (unsigned int t = 0; t < numElements; t++) + { + if (row[t] > 0.0f) + { + float gap = ts[t] - prev; + if (gap > maxGap) maxGap = gap; + prev = ts[t]; + } + } + maxPeriods[r] = maxGap; + } + + ''', 'periodKernel') + + def _creatingItemSets(self): + """ + Storing the complete transactions of the database/input file in a database variable + """ + self._transactions, self._fuzzyValues, self._ts = [], [], [] + if isinstance(self._iFile, _ab._pd.DataFrame): + if self._iFile.empty: + print("its empty..") + i = self._iFile.columns.values.tolist() + if 'TS' in i: + self._ts = self._iFile['TS'].tolist() + if 'Transactions' in i: + self._transactions = self._iFile['Transactions'].tolist() + if 'fuzzyValues' in i: + self._fuzzyValues = self._iFile['fuzzyValues'].tolist() + if isinstance(self._iFile, str): + if _ab._validators.url(self._iFile): + data = _ab._urlopen(self._iFile) + for line in data: + line = line.decode("utf-8") + line = line.split("\n")[0] + parts = line.split(":") + parts[0] = parts[0].strip() + parts[1] = parts[1].strip() + items = [x for x in parts[0].split(self._sep) if x] + quantities = [float(x) for x in parts[1].split(self._sep) if x] + self._ts.append(int(items[0])) + self._transactions.append(items[1:]) + self._fuzzyValues.append(quantities) + else: + try: + with open(self._iFile, 'r', encoding='utf-8') as f: + for line in f: + line = line.split("\n")[0] + parts = line.split(":") + parts[0] = parts[0].strip() + parts[1] = parts[1].strip() + items = [x for x in parts[0].split(self._sep) if x] + quantities = [float(x) for x in parts[1].split(self._sep) if x] + self._ts.append(int(items[0])) + self._transactions.append(items[1:]) + self._fuzzyValues.append(quantities) + except IOError: + print("File Not Found") + quit() + + def _convert(self, value): + """ + + To convert the user specified maxPer value + + :param value: user specified maxPer value + + :type value: int or float or str + + :return: converted type + + """ + if type(value) is int: + value = int(value) + if type(value) is float: + value = (self._dbLen * value) + if type(value) is str: + if '.' in value: + value = float(value) + value = (self._dbLen * value) + else: + value = int(value) + return value + + def arraysAndItems(self): + """ + Builds the fuzzy value array of every item, computes the support and the maximum + periodicity of the single items, and returns the arrays of the items that satisfy minSup. + """ + ArraysAndItems = {} + + for i in range(self._dbLen): + for item, fuzzyValue in zip(self._transactions[i], self._fuzzyValues[i]): + j = tuple([item]) + if j not in ArraysAndItems: + ArraysAndItems[j] = _ab._np.zeros(self._dbLen, dtype=_ab._np.float32) + ArraysAndItems[j][i] = fuzzyValue + + lastTs = self._ts[-1] + newArraysAndItems = {} + + for k, v in ArraysAndItems.items(): + support = float(v.sum()) + if support >= self._minSup: + maxPeriod = 0 + prev = None + for idx in _ab._np.nonzero(v)[0]: + t = self._ts[idx] + if prev is not None and t != lastTs: + maxPeriod = max(maxPeriod, t - prev) + prev = t + if maxPeriod <= self._maxPer: + self._finalPatterns[k] = [support, maxPeriod] + newArraysAndItems[k] = _ab._cp.array(v) + + return newArraysAndItems + + def _maxPeriods(self, matrix): + """ + Computes the maximum periodicity of every row of a fuzzy value matrix on the GPU. + A row's periods are the gaps between the timestamps of its consecutive non-zero + entries, including the gap from time 0 to the first occurrence. + + :param matrix: cupy matrix of fuzzy values, one candidate pattern per row + :return: numpy array with the maximum period of each row + """ + numRows = matrix.shape[0] + tsArr = _ab._cp.asarray(self._ts, dtype=_ab._cp.float32) + maxPeriods = _ab._cp.zeros(numRows, dtype=_ab._cp.float32) + threads = 256 + blocks = (numRows + threads - 1) // threads + self._periodKernel((blocks,), (threads,), + (matrix, tsArr, maxPeriods, + _ab._np.uint32(self._dbLen), _ab._np.uint32(numRows))) + return maxPeriods.get() + + @deprecated("It is recommended to use 'mine()' instead of 'startMine()' for mining process. Starting from January 2025, 'startMine()' will be completely terminated.") + def startMine(self): + """ + Fuzzy periodic frequent pattern mining process will start from here + """ + self.mine() + + def mine(self): + """ + Fuzzy periodic frequent pattern mining process will start from here + """ + _ab._cp.cuda.Device(0).use() + self._startTime = _ab._time.time() + self._creatingItemSets() + self._dbLen = len(self._transactions) + self._minSup = float(self._minSup) + self._maxPer = self._convert(self._maxPer) + + ArraysAndItems = self.arraysAndItems() + + while len(ArraysAndItems) > 0: + newArraysAndItems = {} + keys = list(ArraysAndItems.keys()) + + pairsA, pairsB, unions = [], [], [] + seen = set() + for i in range(len(ArraysAndItems)): + iList = list(keys[i]) + for j in range(i + 1, len(ArraysAndItems)): + jList = list(keys[j]) + union = tuple(sorted(set(iList + jList))) + if len(union) != len(iList) + 1 or union in seen: + continue + seen.add(union) + pairsA.append(i) + pairsB.append(j) + unions.append(union) + + if len(pairsA) > 0: + numPairs = len(pairsA) + matrix = _ab._cp.stack(list(ArraysAndItems.values())) + pairsA = _ab._np.asarray(pairsA, dtype=_ab._np.uint32) + pairsB = _ab._np.asarray(pairsB, dtype=_ab._np.uint32) + supports = _ab._cp.zeros(numPairs, dtype=_ab._cp.float32) + self._supportKernel((numPairs,), (256,), + (matrix, _ab._cp.array(pairsA), _ab._cp.array(pairsB), supports, + _ab._np.uint32(self._dbLen), _ab._np.uint32(numPairs))) + supports = supports.get() + + survivors = _ab._np.where(supports >= self._minSup)[0] + if len(survivors) > 0: + survA = _ab._cp.array(pairsA[survivors]) + survB = _ab._cp.array(pairsB[survivors]) + newMatrix = _ab._cp.minimum(matrix[survA], matrix[survB]) + maxPeriods = self._maxPeriods(newMatrix) + for row, idx in enumerate(survivors): + union = unions[idx] + if maxPeriods[row] <= self._maxPer: + self._finalPatterns[union] = [float(supports[idx]), int(maxPeriods[row])] + newArraysAndItems[union] = newMatrix[row] + + ArraysAndItems = newArraysAndItems + + self._endTime = _ab._time.time() + process = _ab._psutil.Process(_ab._os.getpid()) + self._memoryUSS = float() + self._memoryRSS = float() + self._memoryUSS = process.memory_full_info().uss + self._memoryRSS = process.memory_info().rss + print("Fuzzy periodic frequent patterns were generated successfully using cuFPFPMiner algorithm ") + + def getMemoryUSS(self): + """ + Total amount of USS memory consumed by the mining process will be retrieved from this function + :return: returning USS memory consumed by the mining process + :rtype: float + """ + + return self._memoryUSS + + def getMemoryRSS(self): + """ + Total amount of RSS memory consumed by the mining process will be retrieved from this function + :return: returning RSS memory consumed by the mining process + :rtype: float + """ + + return self._memoryRSS + + def getRuntime(self): + """ + Calculating the total amount of runtime taken by the mining process + :return: returning total amount of runtime taken by the mining process + :rtype: float + """ + + return self._endTime - self._startTime + + def getPatternsAsDataFrame(self): + """ + Storing final fuzzy periodic frequent patterns in a dataframe + :return: returning fuzzy periodic frequent patterns in a dataframe + :rtype: pd.DataFrame + """ + + dataFrame = {} + data = [] + for a, b in self._finalPatterns.items(): + data.append([" ".join(a), b[0], b[1]]) + dataFrame = _ab._pd.DataFrame(data, columns=['Patterns', 'Support', 'Periodicity']) + return dataFrame + + def save(self, outFile): + """ + Complete set of fuzzy periodic frequent patterns will be loaded in to an output file + :param outFile: name of the output file + :type outFile: csvfile + """ + self._oFile = outFile + writer = open(self._oFile, 'w+') + for x, y in self._finalPatterns.items(): + s1 = "\t".join(x) + ":" + str(y[0]) + ":" + str(y[1]) + writer.write("%s \n" % s1) + + def getPatterns(self): + """ + Function to send the set of fuzzy periodic frequent patterns after completion of the mining process + :return: returning fuzzy periodic frequent patterns + :rtype: dict + """ + return self._finalPatterns + + def printResults(self): + """ + This function is used to print results + """ + print("Total number of Fuzzy Periodic Frequent Patterns:", len(self.getPatterns())) + print("Total Memory in USS:", self.getMemoryUSS()) + print("Total Memory in RSS", self.getMemoryRSS()) + print("Total ExecutionTime in s:", self.getRuntime()) + +if __name__ == "__main__": + _ap = str() + if len(_ab._sys.argv) == 5 or len(_ab._sys.argv) == 6: + if len(_ab._sys.argv) == 6: + _ap = cuFPFPMiner(_ab._sys.argv[1], _ab._sys.argv[3], _ab._sys.argv[4], _ab._sys.argv[5]) + if len(_ab._sys.argv) == 5: + _ap = cuFPFPMiner(_ab._sys.argv[1], _ab._sys.argv[3], _ab._sys.argv[4]) + _ap.mine() + print("Total number of Fuzzy Periodic Frequent Patterns:", len(_ap.getPatterns())) + _ap.save(_ab._sys.argv[2]) + print("Total Memory in USS:", _ap.getMemoryUSS()) + print("Total Memory in RSS", _ap.getMemoryRSS()) + print("Total ExecutionTime in s:", _ap.getRuntime()) + else: + print("Error! The number of input parameters do not match the total number of parameters provided") diff --git a/setup.py b/setup.py index af134b5e..aa858fee 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='pami', - version='2026.07.09.1', + version='2026.07.15.1', author='Rage Uday Kiran', author_email='uday.rage@gmail.com', description='This software is being developed at the University of Aizu, Aizu-Wakamatsu, Fukushima, Japan',