Where the Bugs are¶

from bookutils import YouTubeVideo
YouTubeVideo("Aifq0JOc1Jc")

import bookutils.setup

import Tracking

from pydriller import Repository  # https://pydriller.readthedocs.io/

from pydriller.domain.commit import Commit

from pydriller.domain.commit import ModifiedFile

import os

def current_repo() -> Optional[str]:
    path = os.getcwd()
    while True:
        if os.path.exists(os.path.join(path, '.git')):
            return os.path.normpath(path)

        # Go one level up
        new_path = os.path.normpath(os.path.join(path, '..'))
        if new_path != path:
            path = new_path
        else:
            return None

    return None

current_repo()

from datetime import datetime

book_miner = Repository(current_repo(), to=datetime(2020, 10, 1))

DEBUGGINGBOOK_REMOTE_REPO = 'https://github.com/uds-se/debuggingbook.git'
# book_miner = Repository(DEBUGGINGBOOK_REMOTE_REPO)

book_commits = book_miner.traverse_commits()
book_first_commit = next(book_commits)

[attr for attr in dir(book_first_commit) if not attr.startswith('_')]

book_first_commit.msg

[attr for attr in dir(book_first_commit.author) if not attr.startswith('_')]

book_first_commit.author.name, book_first_commit.author.email

book_first_commit.modified_files

[attr for attr in dir(book_first_commit.modified_files[0]) if not attr.startswith('_')]

book_first_commit.modified_files[0].new_path

print(book_first_commit.modified_files[0].content)

print(book_first_commit.modified_files[0].content_before)

book_second_commit = next(book_commits)

[m.new_path for m in book_second_commit.modified_files]

readme_modification = [m for m in book_second_commit.modified_files if m.new_path == 'README.md'][0]

print(str(readme_modification.content_before, 'utf8'))

print(str(readme_modification.content[:400], 'utf8'))

print(readme_modification.diff[:100])

readme_modification.diff_parsed['added'][:10]

tuple('debuggingbook/notebooks/ChangeCounter.ipynb'.split('/'))

Node = Tuple

from collections import defaultdict

import warnings

from git.exc import GitCommandError

class ChangeCounter:
    """Count the number of changes for a repository."""

    def __init__(self, repo: str, *, 
                 filter: Optional[Callable[[Commit], bool]] = None, 
                 log: bool = False, 
                 **kwargs: Any) -> None:
        """
        Constructor.
        `repo` is a git repository (as URL or directory).
        `filter` is a predicate that takes a modification and returns True 
        if it should be considered (default: consider all).
        `log` turns on logging if set.
        `kwargs` are passed to the `Repository()` constructor.
        """
        self.repo = repo
        self.log = log

        if filter is None:
            def filter(m: ModifiedFile) -> bool:
                return True
        assert filter is not None

        self.filter = filter

        # A node is an tuple (f_1, f_2, f_3, ..., f_n) denoting
        # a folder f_1 holding a folder f_2 ... holding a file f_n.

        # Mapping node -> #of changes
        self.changes: Dict[Node, int] = defaultdict(int)

        # Mapping node -> list of commit messages
        self.messages: Dict[Node, List[str]] = defaultdict(list)

        # Mapping node -> last size seen
        self.sizes: Dict[Node, Union[int, float]] = {}

        self.mine(**kwargs)

class ChangeCounter(ChangeCounter):
    def mine(self, **kwargs: Any) -> None:
        """Gather data from repository. To be extended in subclasses."""
        miner = Repository(self.repo, **kwargs)

        for commit in miner.traverse_commits():
            try:
                self.mine_commit(commit)
            except GitCommandError as err:
                # Warn about failing git commands, but continue
                warnings.warn("Cannot mine commit " + repr(commit.hash) + '\n' + str(err))
            except (ValueError, TypeError) as err:
                warnings.warn("Cannot mine commit " + repr(commit.hash) + '\n' + str(err))
                raise err

    def mine_commit(self, commit: Commit) -> None:
        for m in commit.modified_files:
            m.committer = commit.committer
            m.committer_date = commit.committer_date
            m.msg = commit.msg

            if self.include(m):
                self.update_stats(m)

class ChangeCounter(ChangeCounter):
    def include(self, m: ModifiedFile) -> bool:
        """
        Return True if the modification `m` should be included
        (default: the `filter` predicate given to the constructor).
        To be overloaded in subclasses.
        """
        return self.filter(m)

class ChangeCounter(ChangeCounter):
    def update_stats(self, m: ModifiedFile) -> None:
        """
        Update counters with modification `m`.
        Can be extended in subclasses.
        """
        if not m.new_path:
            return

        node = tuple(m.new_path.split('/'))

        self.update_size(node, len(m.content) if m.content else 0)
        self.update_changes(node, m.msg)

        self.update_elems(node, m)

class ChangeCounter(ChangeCounter):
    def update_size(self, node: Tuple, size: int) -> None:
        """
        Update counters for `node` with `size`.
        Can be extended in subclasses.
        """
        self.sizes[node] = size

class ChangeCounter(ChangeCounter):
    def update_changes(self, node: Tuple, commit_msg: str) -> None:
        """
        Update stats for `node` changed with `commit_msg`.
        Can be extended in subclasses.
        """
        self.changes[node] += 1

        self.messages[node].append(commit_msg)

class ChangeCounter(ChangeCounter):
    def update_elems(self, node: Tuple, m: ModifiedFile) -> None:
        """
        Update counters for subelements of `node` with modification `m`.
        To be defined in subclasses.
        """
        pass

DEBUGGINGBOOK_REPO = current_repo()

DEBUGGINGBOOK_REPO

DEBUGGINGBOOK_START_DATE: datetime = datetime(2021, 3, 1)

NUM_WORKERS = 4  # Number of threads to be run in parallel

def debuggingbook_change_counter(
        cls: Type,
        start_date: datetime = DEBUGGINGBOOK_START_DATE) -> Any:
    """
    Instantiate a ChangeCounter (sub)class `cls` with the debuggingbook repo.
    Only mines changes after `start_date` (default: DEBUGGINGBOOK_START_DATE)
    """

    def filter(m: ModifiedFile) -> bool:
        """
        Do not include
        * the `docs/` directory; it only holds generated Web pages
        * the `notebooks/shared/` package; this is infrastructure
        * the `synopsis` pictures; these are all generated
         """
        return (m.new_path and
                not m.new_path.startswith('docs/') and
                not m.new_path.startswith('notebooks/shared/') and
                '-synopsis-' not in m.new_path)

    return cls(DEBUGGINGBOOK_REPO,
               filter=filter,
               since=start_date,
               num_workers=NUM_WORKERS)

from Timer import Timer

with Timer() as t:
    change_counter = debuggingbook_change_counter(ChangeCounter)

t.elapsed_time()

list(change_counter.changes.keys())[:10]

change_counter.changes.get(('Chapters.makefile',), None)

change_counter.messages.get(('Chapters.makefile',), None)

for node in change_counter.changes:
    assert len(change_counter.messages[node]) == change_counter.changes[node]

change_counter.sizes.get(('Chapters.makefile',), None)

import easyplotly as ep
import plotly.graph_objects as go

import math

class ChangeCounter(ChangeCounter):
    def map_node_sizes(self,scale: str = 'log') -> \
        Dict[Node, Union[int, float]]:
        """
        Return a mapping of nodes to sizes.
        Can be overloaded in subclasses.
        """

        if scale == 'log':
            # Default: use log scale
            return {node: math.log(size+1) 
                    for node, size in self.sizes.items()}

        elif scale == 'sqrt':
            # Alternative: use sqrt size
            return {node: math.sqrt(size)
                    for node, size in self.sizes.items()}

        elif scale == 'abs':
            # Alternative: use absolute size
            return self.sizes

        else:
            raise ValueError(f"Unknown scale: {scale}; "
                             f"use one of [log, sqrt, abs]")

class ChangeCounter(ChangeCounter):
    def map_node_color(self, node: Node) -> Optional[int]:
        """
        Return a color of the node, as a number.
        Can be overloaded in subclasses.
        """
        return self.changes.get(node)

class ChangeCounter(ChangeCounter):
    def map_node_text(self, node: Node) -> Optional[str]:
        """
        Return the text to be shown for the node (default: #changes).
        Can be overloaded in subclasses.
        """
        change = self.changes.get(node)
        return str(change) if change is not None else None

class ChangeCounter(ChangeCounter):
    def map_hoverinfo(self) -> str:
        """
        Return the text to be shown when hovering over a node.
        To be overloaded in subclasses.
        """
        return 'label+text'

    def map_colorscale(self) -> str:
        """
        Return the colorscale for the map. To be overloaded in subclasses.
        """
        return 'YlOrRd'

class ChangeCounter(ChangeCounter):
    def map(self) -> go.Figure:
        """Produce an interactive tree map of the repository."""
        treemap = ep.Treemap(
                     self.map_node_sizes(),
                     text=self.map_node_text,
                     hoverinfo=self.map_hoverinfo(),
                     marker_colors=self.map_node_color,
                     marker_colorscale=self.map_colorscale(),
                     root_label=self.repo,
                     branchvalues='total'
                    )

        fig = go.Figure(treemap)
        fig.update_layout(margin=dict(l=0, r=0, t=30, b=0))

        return fig

change_counter = debuggingbook_change_counter(ChangeCounter)

change_counter.map()