Upload TMIDIX.py

TMIDIX.py

@@ -5,9 +5,8 @@ r'''############################################################################
 #
 #
 #	Tegridy MIDI X Module (TMIDI X / tee-midi eks)
-#	Version 1.0
 #
-#   NOTE: TMIDI X Module starts after the partial MIDI.py module @ line 1438
+#   NOTE: TMIDI X Module starts after the partial MIDI.py module @ line 1450
 #
 #	Based upon MIDI.py module v.6.7. by Peter Billam / pjb.com.au
 #
@@ -26,7 +25,7 @@ r'''############################################################################
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 #   Unless required by applicable law or agreed to in writing, software
 #   distributed under the License is distributed on an "AS IS" BASIS,
@@ -47,7 +46,20 @@ r'''############################################################################
 #	Copyright 2020 Peter Billam
 #
 ###################################################################################
-###################################################################################'''
+###################################################################################
+'''
+
+###################################################################################
+
+__version__ = "25.7.8"
+
+print('=' * 70)
+print('TMIDIX Python module')
+print('Version:', __version__)
+print('=' * 70)
+print('Loading module...')
+
+###################################################################################
 
 import sys, struct, copy
 
@@ -1440,7 +1452,6 @@ def _encode(events_lol, unknown_callback=None, never_add_eot=False,
 ###################################################################################
 #
 #	Tegridy MIDI X Module (TMIDI X / tee-midi eks)
-#	Version 1.0
 #
 #	Based upon and includes the amazing MIDI.py module v.6.7. by Peter Billam
 #	pjb.com.au
@@ -1477,6 +1488,7 @@ from itertools import groupby
 
 from collections import Counter
 from collections import defaultdict
+from collections import OrderedDict
 
 from operator import itemgetter
 
@@ -1497,6 +1509,13 @@ from pathlib import Path
 
 import shutil
 
+import hashlib
+
+from array import array
+
+from pathlib import Path
+from fnmatch import fnmatch
+
 ###################################################################################
 #
 # Original TMIDI Tegridy helper functions
@@ -3705,19 +3724,52 @@ def validate_pitches(chord, channel_to_check = 0, return_sorted = True):
       chord.sort(key = lambda x: x[4], reverse=True)
       return chord
 
-def adjust_score_velocities(score, max_velocity):
+def adjust_score_velocities(score,
+                            max_velocity,
+                            adj_per_channel=False,
+                            adj_in_place=True
+                           ):
+  
+    if adj_in_place:
+        buf = score
+        
+    else:
+        buf = copy.deepcopy(score)
+
+    notes = [evt for evt in buf if evt[0] == 'note']
+    
+    if not notes:
+        return buf
+
+    if adj_per_channel:
+        channel_max = {}
+        
+        for _, _, _, ch, _, vel, _ in notes:
+            channel_max[ch] = max(channel_max.get(ch, 0), vel)
 
-    min_velocity = min([c[5] for c in score])
-    max_velocity_all_channels = max([c[5] for c in score])
-    min_velocity_ratio = min_velocity / max_velocity_all_channels
+        channel_factor = {
+            ch: (max_velocity / vmax if vmax > 0 else 1.0)
+            for ch, vmax in channel_max.items()
+        }
+
+        for evt in buf:
+            if evt[0] == 'note':
+                ch = evt[3]
+                factor = channel_factor.get(ch, 1.0)
+                new_vel = int(evt[5] * factor)
+                evt[5] = max(1, min(127, new_vel))
 
-    max_channel_velocity = max([c[5] for c in score])
-    if max_channel_velocity < min_velocity:
-        factor = max_velocity / min_velocity
     else:
-        factor = max_velocity / max_channel_velocity
-    for i in range(len(score)):
-        score[i][5] = int(score[i][5] * factor)
+        global_max = max(vel for _, _, _, _, _, vel, _ in notes)
+        factor = max_velocity / global_max if global_max > 0 else 1.0
+
+        for evt in buf:
+            if evt[0] == 'note':
+                new_vel = int(evt[5] * factor)
+                evt[5] = max(1, min(127, new_vel))
+
+    if not adj_in_place:
+        return buf
 
 def chordify_score(score,
                   return_choridfied_score=True,
@@ -3849,7 +3901,10 @@ def chordify_score(score,
     else:
       return None
 
-def fix_monophonic_score_durations(monophonic_score):
+def fix_monophonic_score_durations(monophonic_score,
+                                   min_notes_gap=1,
+                                   min_notes_dur=1
+                                   ):
   
     fixed_score = []
 
@@ -3861,15 +3916,17 @@ def fix_monophonic_score_durations(monophonic_score):
         nmt = monophonic_score[i+1][1]
 
         if note[1]+note[2] >= nmt:
-          note_dur = nmt-note[1]-1
+          note_dur = max(1, nmt-note[1]-min_notes_gap)
         else:
           note_dur = note[2]
 
         new_note = [note[0], note[1], note_dur] + note[3:]
-
-        fixed_score.append(new_note)
-
-      fixed_score.append(monophonic_score[-1])
+        
+        if new_note[2] >= min_notes_dur:
+            fixed_score.append(new_note)
+      
+      if monophonic_score[-1][2] >= min_notes_dur:
+          fixed_score.append(monophonic_score[-1])
 
     elif type(monophonic_score[0][0]) == int:
 
@@ -3879,15 +3936,17 @@ def fix_monophonic_score_durations(monophonic_score):
         nmt = monophonic_score[i+1][0]
 
         if note[0]+note[1] >= nmt:
-          note_dur = nmt-note[0]-1
+          note_dur = max(1, nmt-note[0]-min_notes_gap)
         else:
           note_dur = note[1]
-
+          
         new_note = [note[0], note_dur] + note[2:]
-
-        fixed_score.append(new_note)
-
-      fixed_score.append(monophonic_score[-1]) 
+        
+        if new_note[1] >= min_notes_dur:
+            fixed_score.append(new_note)
+      
+      if monophonic_score[-1][1] >= min_notes_dur:
+          fixed_score.append(monophonic_score[-1]) 
 
     return fixed_score
 
@@ -5003,54 +5062,101 @@ def patch_list_from_enhanced_score_notes(enhanced_score_notes,
 
 ###################################################################################
 
-def patch_enhanced_score_notes(enhanced_score_notes, 
-                                default_patch=0, 
-                                drums_patch=9,
-                                verbose=False
-                                ):
+def patch_enhanced_score_notes(escore_notes,
+                               default_patch=0,
+                               reserved_patch=-1,
+                               reserved_patch_channel=-1,
+                               drums_patch=9,
+                               verbose=False
+                              ):
   
-    #===========================================================================    
+    #===========================================================================
+
+    enhanced_score_notes = copy.deepcopy(escore_notes)
+
+    #===========================================================================
   
     enhanced_score_notes_with_patch_changes = []
 
     patches = [-1] * 16
 
+    if -1 < reserved_patch < 128 and -1 < reserved_patch_channel < 128:
+        patches[reserved_patch_channel] = reserved_patch
+
     overflow_idx = -1
 
     for idx, e in enumerate(enhanced_score_notes):
-      if e[0] == 'note':
-        if e[3] != 9:
-            if patches[e[3]] == -1:
-                patches[e[3]] = e[6]
-            else:
-                if patches[e[3]] != e[6]:
-                  if e[6] in patches:
-                    e[3] = patches.index(e[6])
-                  else:
-                    if -1 in patches:
-                        patches[patches.index(-1)] = e[6]
-                    else:
-                        overflow_idx = idx
-                        break
-
-      enhanced_score_notes_with_patch_changes.append(e)
+        if e[0] == 'note':
+            if e[3] != 9:
+                if -1 < reserved_patch < 128 and -1 < reserved_patch_channel < 128:
+                    if e[6] == reserved_patch:
+                        e[3] = reserved_patch_channel
+                    
+                if patches[e[3]] == -1:
+                    patches[e[3]] = e[6]
+                
+                else:
+                    if patches[e[3]] != e[6]:
+                        if e[6] in patches:
+                            e[3] = patches.index(e[6])
+                        
+                        else:
+                            if -1 in patches:
+                                patches[patches.index(-1)] = e[6]
+                            
+                            else:
+                                overflow_idx = idx
+                                break
+
+        enhanced_score_notes_with_patch_changes.append(e)
 
     #===========================================================================
 
     overflow_patches = []
+    overflow_channels = [-1] * 16
+    overflow_channels[9] = drums_patch
+
+    if -1 < reserved_patch < 128 and -1 < reserved_patch_channel < 128:
+        overflow_channels[reserved_patch_channel] = reserved_patch
 
     if overflow_idx != -1:
-      for idx, e in enumerate(enhanced_score_notes[overflow_idx:]):
-        if e[0] == 'note':
-          if e[3] != 9:
-            if e[6] not in patches:
-              if e[6] not in overflow_patches:
-                overflow_patches.append(e[6])
-                enhanced_score_notes_with_patch_changes.append(['patch_change', e[1], e[3], e[6]])
-            else:
-              e[3] = patches.index(e[6])
+        for idx, e in enumerate(enhanced_score_notes[overflow_idx:]):
+            if e[0] == 'note':
+                if e[3] != 9:
+                    if e[6] not in overflow_channels:
+                        
+                        if -1 in overflow_channels:
+                            free_chan = overflow_channels.index(-1)
+                            overflow_channels[free_chan] = e[6]
+                            e[3] = free_chan
+
+                            enhanced_score_notes_with_patch_changes.append(['patch_change', e[1], e[3], e[6]])
+                            
+                            overflow_patches.append(e[6])
+                
+                        else:
+                            overflow_channels = [-1] * 16
+                            overflow_channels[9] = drums_patch
+                            
+                            if -1 < reserved_patch < 128 and -1 < reserved_patch_channel < 128:
+                                overflow_channels[reserved_patch_channel] = reserved_patch
+                                e[3] = reserved_patch_channel
+
+                            if e[6] != reserved_patch:
+
+                                free_chan = overflow_channels.index(-1)
+                                e[3] = free_chan
+                                    
+                            overflow_channels[e[3]] = e[6]                            
+
+                            enhanced_score_notes_with_patch_changes.append(['patch_change', e[1], e[3], e[6]])
+                            
+                            overflow_patches.append(e[6])
+
+                    else:
+                        e[3] = overflow_channels.index(e[6])
 
-          enhanced_score_notes_with_patch_changes.append(e)
+            enhanced_score_notes_with_patch_changes.append(e)
 
     #===========================================================================
 
@@ -5060,9 +5166,13 @@ def patch_enhanced_score_notes(enhanced_score_notes,
 
     #===========================================================================
 
+    overflow_patches = ordered_set(overflow_patches)
+
+    #===========================================================================
+
     if verbose:
       print('=' * 70)
-      print('Composition patches')
+      print('Main composition patches')
       print('=' * 70)
       for c, p in enumerate(patches):
         print('Cha', str(c).zfill(2), '---', str(p).zfill(3), Number2patch[p])
@@ -5075,6 +5185,8 @@ def patch_enhanced_score_notes(enhanced_score_notes,
           print(str(p).zfill(3), Number2patch[p])
         print('=' * 70)
 
+    #===========================================================================
+
     return enhanced_score_notes_with_patch_changes, patches, overflow_patches
 
 ###################################################################################
@@ -11170,13 +11282,17 @@ def escore_notes_core(escore_notes, core_len=128):
 
 ###################################################################################
 
-def multiprocessing_wrapper(function, data_list):
+def multiprocessing_wrapper(function, data_list, verbose=True):
     
     with multiprocessing.Pool() as pool:
         
         results = []
         
-        for result in tqdm.tqdm(pool.imap_unordered(function, data_list), total=len(data_list)):
+        for result in tqdm.tqdm(pool.imap_unordered(function, data_list),
+                                total=len(data_list),
+                                disable=not verbose
+                                ):
+            
             results.append(result)
             
     return results
@@ -11325,28 +11441,83 @@ def system_memory_utilization(return_dict=False):
 
 ###################################################################################
 
+def system_cpus_utilization(return_dict=False):
+
+    if return_dict:
+        return {'num_cpus': psutil.cpu_count(),
+                'cpus_util': psutil.cpu_percent()
+                }
+
+    else:
+        print('Number of CPUs:', psutil.cpu_count())
+        print('CPUs utilization:', psutil.cpu_percent())
+
+###################################################################################
+
 def create_files_list(datasets_paths=['./'],
                       files_exts=['.mid', '.midi', '.kar', '.MID', '.MIDI', '.KAR'],
+                      max_num_files_per_dir=-1,
+                      randomize_dir_files=False,
+                      max_total_files=-1,
                       randomize_files_list=True,
+                      check_for_dupes=False,
+                      use_md5_hashes=False,
+                      return_dupes=False,
                       verbose=True
                      ):
+    
     if verbose:
         print('=' * 70)
         print('Searching for files...')
         print('This may take a while on a large dataset in particular...')
         print('=' * 70)
 
-    filez_set = defaultdict(None)
-
     files_exts = tuple(files_exts)
     
-    for dataset_addr in tqdm.tqdm(datasets_paths, disable=not verbose):
-        for dirpath, dirnames, filenames in os.walk(dataset_addr):
-            for file in filenames:
-                if file not in filez_set and file.endswith(files_exts):
-                    filez_set[os.path.join(dirpath, file)] = None
-    
-    filez = list(filez_set.keys())
+    filez_set = defaultdict(None)
+    dupes_list = []
+ 
+    for dataset_addr in datasets_paths:
+        
+        print('=' * 70)
+        print('Processing', dataset_addr)
+        print('=' * 70)
+        
+        for dirpath, dirnames, filenames in tqdm.tqdm(os.walk(dataset_addr), disable=not verbose):
+                
+                if randomize_dir_files:
+                    random.shuffle(filenames)
+                    
+                if max_num_files_per_dir > 0:
+                    max_num_files = max_num_files_per_dir
+                    
+                else:
+                    max_num_files = len(filenames)
+                    
+                for file in filenames[:max_num_files]:
+                    if file.endswith(files_exts):
+                        if check_for_dupes:
+                        
+                            if use_md5_hashes:
+                                md5_hash = hashlib.md5(open(os.path.join(dirpath, file), 'rb').read()).hexdigest()
+                                
+                                if md5_hash not in filez_set:
+                                    filez_set[md5_hash] = os.path.join(dirpath, file)
+                                
+                                else:
+                                    dupes_list.append(os.path.join(dirpath, file))
+                                    
+                            else:
+                                if file not in filez_set:
+                                    filez_set[file] = os.path.join(dirpath, file)
+                                
+                                else:
+                                    dupes_list.append(os.path.join(dirpath, file))
+                        else:
+                            fpath = os.path.join(dirpath, file)
+                            filez_set[fpath] = fpath                              
+
+    filez = list(filez_set.values())
 
     if verbose:
         print('Done!')
@@ -11366,6 +11537,7 @@ def create_files_list(datasets_paths=['./'],
                 
         if verbose:
             print('Found', len(filez), 'files.')
+            print('Skipped', len(dupes_list), 'duplicate files.')
             print('=' * 70)
  
     else:
@@ -11373,8 +11545,20 @@ def create_files_list(datasets_paths=['./'],
             print('Could not find any files...')
             print('Please check dataset dirs and files extensions...')
             print('=' * 70)
+    
+    if max_total_files > 0:     
+        if return_dupes:
+            return filez[:max_total_files], dupes_list
         
-    return filez
+        else:
+            return filez[:max_total_files]
+    
+    else:
+        if return_dupes:
+            return filez, dupes_list
+        
+        else:
+            return filez
 
 ###################################################################################
 
@@ -12196,6 +12380,12 @@ def escore_notes_pitches_chords_signature(escore_notes,
                                           use_full_chords=False
                                          ):
     
+    if use_full_chords:
+        CHORDS = ALL_CHORDS_FULL
+        
+    else:
+        CHORDS = ALL_CHORDS_SORTED
+    
     max_patch = max(0, min(128, max_patch))
 
     escore_notes = [e for e in escore_notes if e[6] <= max_patch]
@@ -12207,7 +12397,7 @@ def escore_notes_pitches_chords_signature(escore_notes,
         sig = []
         dsig = []
         
-        drums_offset = 321 + 128
+        drums_offset = len(CHORDS) + 128
         
         bad_chords_counter = 0
         
@@ -12224,10 +12414,10 @@ def escore_notes_pitches_chords_signature(escore_notes,
                 tones_chord = sorted(set([p % 12 for p in pitches]))
                      
                 try:
-                    sig_token = ALL_CHORDS_SORTED.index(tones_chord) + 128
+                    sig_token = CHORDS.index(tones_chord) + 128
                 except:
                     checked_tones_chord = check_and_fix_tones_chord(tones_chord, use_full_chords=use_full_chords)
-                    sig_token = ALL_CHORDS_SORTED.index(checked_tones_chord) + 128
+                    sig_token = CHORDS.index(checked_tones_chord) + 128
                     bad_chords_counter += 1
                     
               elif len(pitches) == 1:
@@ -12438,10 +12628,10 @@ def merge_chords(chord1, chord2, sort_drums_last=False):
         e[1] = mchord[0][1]
     
     if sort_drums_last:
-        mchord.sort(key=lambda x: (x[4], x[6]) if x[6] != 128 else (x[6], -x[4]))
+        mchord.sort(key=lambda x: (-x[4], x[6]) if x[6] != 128 else (x[6], -x[4]))
 
     else:
-        mchord.sort(key=lambda x: (x[4], x[6]))
+        mchord.sort(key=lambda x: (-x[4], x[6]))
 
     return mchord
     
@@ -12753,6 +12943,672 @@ def multi_instrumental_escore_notes_tokenized(escore_notes, compress_seq=False):
 
     return melody_chords
 
+###################################################################################
+
+def merge_counts(data, return_lists=True):
+    
+    merged = defaultdict(int)
+    
+    for value, count in data:
+        merged[value] += count
+
+    if return_lists:
+        return [[k, v] for k, v in merged.items()]
+
+    else:
+        return list(merged.items())
+    
+###################################################################################
+
+def convert_escore_notes_pitches_chords_signature(signature, convert_to_full_chords=True):
+
+    if convert_to_full_chords:
+        SRC_CHORDS = ALL_CHORDS_SORTED
+        TRG_CHORDS = ALL_CHORDS_FULL
+
+    else:
+        SRC_CHORDS = ALL_CHORDS_FULL
+        TRG_CHORDS = ALL_CHORDS_SORTED
+
+    cdiff = len(TRG_CHORDS) - len(SRC_CHORDS)
+
+    pitches_counts = [c for c in signature if -1 < c[0] < 128]
+    chords_counts = [c for c in signature if 127 < c[0] < len(SRC_CHORDS)+128]
+    drums_counts = [[c[0]+cdiff, c[1]] for c in signature if len(SRC_CHORDS)+127 < c[0] < len(SRC_CHORDS)+256]
+    bad_chords_count = [c for c in signature if c[0] == -1]
+
+    new_chords_counts = []
+    
+    for c in chords_counts:
+        tones_chord = SRC_CHORDS[c[0]-128]
+
+        if tones_chord not in TRG_CHORDS:
+            tones_chord = check_and_fix_tones_chord(tones_chord, use_full_chords=convert_to_full_chords)
+            bad_chords_count[0][1] += 1
+            
+        new_chords_counts.append([TRG_CHORDS.index(tones_chord)+128, c[1]])
+
+    return pitches_counts + merge_counts(new_chords_counts) + drums_counts + bad_chords_count
+
+###################################################################################
+
+def convert_bytes_in_nested_list(lst, encoding='utf-8', errors='ignore'):
+    
+    new_list = []
+    
+    for item in lst:
+        if isinstance(item, list):
+            new_list.append(convert_bytes_in_nested_list(item))
+            
+        elif isinstance(item, bytes):
+            new_list.append(item.decode(encoding, errors=errors))
+            
+        else:
+            new_list.append(item)
+            
+    return new_list
+
+###################################################################################
+
+def mult_pitches(pitches, min_oct=4, max_oct=6):
+    
+    tones_chord = sorted(set([p % 12 for p in pitches]))
+
+    mult_ptcs = []
+
+    for t in tones_chord:
+        for i in range(min_oct, max_oct):
+            mult_ptcs.append((i*12)+t)
+
+    return mult_ptcs
+
+###################################################################################
+
+def find_next(pitches, cur_ptc):
+
+    i = 0
+
+    for i, p in enumerate(pitches):
+        if p != cur_ptc:
+            break
+
+    return i
+
+###################################################################################
+
+def ordered_groups_unsorted(data, key_index):
+   
+    def keyfunc(sublist):
+        return sublist[key_index]
+    
+    groups = []
+    
+    for key, group in groupby(data, key=keyfunc):
+        groups.append((key, list(group)))
+        
+    return groups
+
+###################################################################################
+
+def ordered_groups(data, ptc_idx, pat_idx):
+    
+    groups = OrderedDict()
+    
+    for sublist in data:
+        key = tuple([sublist[ptc_idx], sublist[pat_idx]])
+        
+        if key not in groups:
+            groups[key] = []
+            
+        groups[key].append(sublist)
+    
+    return list(groups.items())
+
+###################################################################################
+
+def merge_melody_notes(escore_notes, pitches_idx=4, max_dur=255, last_dur=128):
+
+    groups = ordered_groups_unsorted(escore_notes, pitches_idx)
+
+    merged_melody_notes = []
+
+    for i, (k, g) in enumerate(groups[:-1]):
+        
+        if len(g) == 1:
+            merged_melody_notes.extend(g)
+
+        else:
+            dur = min(max_dur, groups[i+1][1][0][1] - g[0][1])
+            
+            merged_melody_notes.append(['note',
+                                        g[0][1],
+                                        dur,
+                                        g[0][3],
+                                        g[0][4],
+                                        g[0][5],
+                                        g[0][6]
+                                       ])
+ 
+    merged_melody_notes.append(['note',
+                                groups[-1][1][0][1],
+                                last_dur,
+                                groups[-1][1][0][3],
+                                groups[-1][1][0][4],
+                                groups[-1][1][0][5],
+                                groups[-1][1][0][6]
+                               ])
+            
+    return merged_melody_notes
+
+###################################################################################
+
+def add_expressive_melody_to_enhanced_score_notes(escore_notes,
+                                                  melody_start_chord=0,
+                                                  melody_prime_pitch=60,
+                                                  melody_step=1,
+                                                  melody_channel=3,
+                                                  melody_patch=40,
+                                                  melody_notes_max_duration=255,
+                                                  melody_last_note_dur=128,
+                                                  melody_clip_max_min_durs=[],
+                                                  melody_max_velocity=120,
+                                                  acc_max_velocity=90,
+                                                  return_melody=False
+                                                  ):
+
+
+    score = copy.deepcopy(escore_notes)
+
+    adjust_score_velocities(score, acc_max_velocity)
+
+    cscore = chordify_score([1000, score])
+
+    melody_pitches = [melody_prime_pitch]
+    
+    for i, c in enumerate(cscore[melody_start_chord:]):
+        
+        if i % melody_step == 0:
+        
+            pitches = [e[4] for e in c if e[3] != 9]
+        
+            if pitches:
+                cptc = find_closest_value(mult_pitches(pitches), melody_pitches[-1])[0]
+                melody_pitches.append(cptc)
+    
+    song_f = []
+    mel_f = []
+    
+    idx = 1
+    
+    for i, c in enumerate(cscore[:-melody_step]):
+        pitches = [e[4] for e in c if e[3] != 9]
+    
+        if pitches and i >= melody_start_chord and i % melody_step == 0:
+            dur = min(cscore[i+melody_step][0][1] - c[0][1], melody_notes_max_duration)
+            
+            mel_f.append(['note', 
+                          c[0][1], 
+                          dur, 
+                          melody_channel, 
+                          60+(melody_pitches[idx] % 24), 
+                          100 + ((melody_pitches[idx] % 12) * 2), 
+                          melody_patch
+                         ])
+            idx += 1
+            
+        song_f.extend(c)
+        
+    song_f.extend(flatten(cscore[-melody_step:]))
+    
+    if len(melody_clip_max_min_durs) == 2:
+        for e in mel_f:
+            if e[2] >= melody_clip_max_min_durs[0]:
+                e[2] = melody_clip_max_min_durs[1]
+
+    adjust_score_velocities(mel_f, melody_max_velocity)
+    
+    merged_melody_notes = merge_melody_notes(mel_f,
+                                             max_dur=melody_notes_max_duration,
+                                             last_dur=melody_last_note_dur
+                                             )
+
+    song_f = sorted(merged_melody_notes + song_f,
+                    key=lambda x: x[1]
+                   )
+
+    if return_melody:
+        return mel_f
+
+    else:
+        return song_f
+
+###################################################################################
+    
+def list_md5_hash(ints_list):
+    
+    arr = array('H', ints_list)
+    binary_data = arr.tobytes()
+    
+    return hashlib.md5(binary_data).hexdigest()
+
+###################################################################################
+
+def fix_escore_notes_durations(escore_notes,
+                               min_notes_gap=1,
+                               min_notes_dur=1,
+                               times_idx=1,
+                               durs_idx=2,
+                               channels_idx = 3, 
+                               pitches_idx=4,
+                               patches_idx=6
+                              ):
+
+    notes = [e for e in escore_notes if e[channels_idx] != 9]
+    drums = [e for e in escore_notes if e[channels_idx] == 9]
+    
+    escore_groups = ordered_groups(notes, pitches_idx, patches_idx)
+
+    merged_score = []
+
+    for k, g in escore_groups:
+        if len(g) > 2:
+            fg = fix_monophonic_score_durations(g, 
+                                                min_notes_gap=min_notes_gap, 
+                                                min_notes_dur=min_notes_dur
+                                               )
+            merged_score.extend(fg)
+
+        elif len(g) == 2:
+
+            if g[0][times_idx]+g[0][durs_idx] >= g[1][times_idx]:
+                g[0][durs_idx] = max(1, g[1][times_idx] - g[0][times_idx] - min_notes_gap)
+                
+            merged_score.extend(g)
+
+        else:
+            merged_score.extend(g)
+
+    return sorted(merged_score + drums, key=lambda x: x[times_idx])
+
+###################################################################################
+
+def create_nested_chords_tree(chords_list):
+    
+    tree = {}
+    
+    for chord in chords_list:
+        
+        node = tree
+        
+        for semitone in chord:
+            if semitone not in node:
+                node[semitone] = {}
+                
+            node = node[semitone]
+            
+        node.setdefault(-1, []).append(chord)
+        
+    return tree
+
+###################################################################################
+
+def get_chords_with_prefix(nested_chords_tree, prefix):
+   
+    node = nested_chords_tree
+    
+    for semitone in prefix:
+        if semitone in node:
+            node = node[semitone]
+            
+        else:
+            return []
+
+    collected_chords = []
+    
+    def recursive_collect(subnode):
+        if -1 in subnode:
+            collected_chords.extend(subnode[-1])
+            
+        for key, child in subnode.items():
+            if key != -1:
+                recursive_collect(child)
+                
+    recursive_collect(node)
+    
+    return collected_chords
+
+###################################################################################
+
+def get_chords_by_semitones(chords_list, chord_semitones):
+
+    query_set = set(chord_semitones)
+    results = []
+
+    for chord in chords_list:
+        
+        chord_set = set(chord)
+        
+        if query_set.issubset(chord_set):
+            results.append(sorted(set(chord)))
+            
+    return results
+
+###################################################################################
+
+def remove_duplicate_pitches_from_escore_notes(escore_notes, 
+                                               pitches_idx=4, 
+                                               patches_idx=6, 
+                                               return_dupes_count=False
+                                              ):
+    
+    cscore = chordify_score([1000, escore_notes])
+
+    new_escore = []
+
+    bp_count = 0
+
+    for c in cscore:
+        
+        cho = []
+        seen = []
+
+        for cc in c:
+            if [cc[pitches_idx], cc[patches_idx]] not in seen:
+                cho.append(cc)
+                seen.append([cc[pitches_idx], cc[patches_idx]])
+
+            else:
+                bp_count += 1
+
+        new_escore.extend(cho)
+        
+    if return_dupes_count:
+        return bp_count
+        
+    else:
+        return new_escore
+
+###################################################################################
+    
+def chunks_shuffle(lst,
+                   min_len=1,
+                   max_len=3,
+                   seed=None
+                   ):
+    
+    rnd = random.Random(seed)
+    chunks = []
+    i, n = 0, len(lst)
+
+    while i < n:
+        size = rnd.randint(min_len, max_len)
+        size = min(size, n - i)
+        chunks.append(lst[i : i + size])
+        i += size
+
+    rnd.shuffle(chunks)
+
+    flattened = []
+    for chunk in chunks:
+        flattened.extend(chunk)
+
+    return flattened
+
+###################################################################################
+
+def convert_bytes_in_nested_list(lst, 
+                                 encoding='utf-8', 
+                                 errors='ignore',
+                                 return_changed_events_count=False
+                                ):
+    
+    new_list = []
+
+    ce_count = 0
+    
+    for item in lst:
+        if isinstance(item, list):
+            new_list.append(convert_bytes_in_nested_list(item))
+            
+        elif isinstance(item, bytes):
+            new_list.append(item.decode(encoding, errors=errors))
+            ce_count += 1
+            
+        else:
+            new_list.append(item)
+            
+    if return_changed_events_count:       
+        return new_list, ce_count
+
+    else:
+        return new_list
+    
+###################################################################################
+    
+def find_deepest_midi_dirs(roots,
+                           marker_file="midi_score.mid",
+                           suffixes=None,
+                           randomize=False,
+                           seed=None,
+                           verbose=False
+                          ):
+    
+    try:
+        iter(roots)
+        if isinstance(roots, (str, Path)):
+            root_list = [roots]
+        else:
+            root_list = list(roots)
+            
+    except TypeError:
+        root_list = [roots]
+
+    if isinstance(marker_file, (list, tuple)):
+        patterns = [p.lower() for p in marker_file if p]
+        
+    else:
+        patterns = [marker_file.lower()] if marker_file else []
+
+    allowed = {s.lower() for s in (suffixes or ['.mid', '.midi', '.kar'])}
+
+    if verbose:
+        print("Settings:")
+        print("  Roots:", [str(r) for r in root_list])
+        print("  Marker patterns:", patterns or "<no marker filter>")
+        print("  Allowed suffixes:", allowed)
+        print(f"  Randomize={randomize}, Seed={seed}")
+
+    results = defaultdict(list)
+    rng = random.Random(seed)
+
+    for root in root_list:
+
+        root_path = Path(root)
+        
+        if not root_path.is_dir():
+            print(f"Warning: '{root_path}' is not a valid directory, skipping.")
+            continue
+
+        if verbose:
+            print(f"\nScanning root: {str(root_path)}")
+
+        all_dirs = list(root_path.rglob("*"))
+        dirs_iter = tqdm.tqdm(all_dirs, desc=f"Dirs in {root_path.name}", disable=not verbose)
+
+        for dirpath in dirs_iter:
+            if not dirpath.is_dir():
+                continue
+
+            children = list(dirpath.iterdir())
+            if any(child.is_dir() for child in children):
+                if verbose:
+                    print(f"Skipping non-leaf: {str(dirpath)}")
+                continue
+
+            files = [f for f in children if f.is_file()]
+            names = [f.name.lower() for f in files]
+
+            if patterns:
+                matched = any(fnmatch(name, pat) for name in names for pat in patterns)
+                if not matched:
+                    if verbose:
+                        print(f"No marker in: {str(dirpath)}")
+                    continue
+                    
+                if verbose:
+                    print(f"Marker found in: {str(dirpath)}")
+                    
+            else:
+                if verbose:
+                    print(f"Including leaf (no marker): {str(dirpath)}")
+
+            for f in files:
+                if f.suffix.lower() in allowed:
+                    results[str(dirpath)].append(str(f))
+                    
+                    if verbose:
+                        print(f"  Collected: {f.name}")
+
+    all_leaves = list(results.keys())
+    if randomize:
+        if verbose:
+            print("\nShuffling leaf directories")
+            
+        rng.shuffle(all_leaves)
+        
+    else:
+        all_leaves.sort()
+
+    final_dict = {}
+    
+    for leaf in all_leaves:
+        file_list = results[leaf][:]
+        if randomize:
+            if verbose:
+                print(f"Shuffling files in: {leaf}")
+                
+            rng.shuffle(file_list)
+            
+        else:
+            file_list.sort()
+            
+        final_dict[leaf] = file_list
+
+    if verbose:
+        print("\nScan complete. Found directories:")
+        for d, fl in final_dict.items():
+            print(f"  {d} -> {len(fl)} files")
+
+    return final_dict
+
+###################################################################################
+
+PERCUSSION_GROUPS = {
+    
+    1: {  # Bass Drums
+        35: 'Acoustic Bass Drum',
+        36: 'Bass Drum 1',
+    },
+    2: {  # Stick
+        37: 'Side Stick',
+    },
+    3: {  # Snares
+        38: 'Acoustic Snare',
+        40: 'Electric Snare',
+    },
+    4: {  # Claps
+        39: 'Hand Clap',
+    },
+    5: {  # Floor Toms
+        41: 'Low Floor Tom',
+        43: 'High Floor Tom',
+    },
+    6: {  # Hi-Hats
+        42: 'Closed Hi-Hat',
+        44: 'Pedal Hi-Hat',
+        46: 'Open Hi-Hat',
+    },
+    7: {  # Toms
+        45: 'Low Tom',
+        47: 'Low-Mid Tom',
+        48: 'Hi-Mid Tom',
+        50: 'High Tom',
+    },
+    8: {  # Cymbals
+        49: 'Crash Cymbal 1',
+        51: 'Ride Cymbal 1',
+        52: 'Chinese Cymbal',
+        55: 'Splash Cymbal',
+        57: 'Crash Cymbal 2',
+        59: 'Ride Cymbal 2',
+    },
+    9: {  # Bells
+        53: 'Ride Bell',
+    },
+    10: {  # Tambourine
+        54: 'Tambourine',
+    },
+    11: {  # Cowbell
+        56: 'Cowbell',
+    },
+    12: {  # Vibraslap
+        58: 'Vibraslap',
+    },
+    13: {  # Bongos
+        60: 'Hi Bongo',
+        61: 'Low Bongo',
+    },
+    14: {  # Congas
+        62: 'Mute Hi Conga',
+        63: 'Open Hi Conga',
+        64: 'Low Conga',
+    },
+    15: {  # Timbales
+        65: 'High Timbale',
+        66: 'Low Timbale',
+    },
+    16: {  # Agogô
+        67: 'High Agogo',
+        68: 'Low Agogo',
+    },
+    17: {  # Cabasa
+        69: 'Cabasa',
+    },
+    18: {  # Maracas
+        70: 'Maracas',
+    },
+    19: {  # Whistles
+        71: 'Short Whistle',
+        72: 'Long Whistle',
+    },
+    20: {  # Guiros
+        73: 'Short Guiro',
+        74: 'Long Guiro',
+    },
+    21: {  # Claves
+        75: 'Claves',
+    },
+    22: {  # Wood Blocks
+        76: 'Hi Wood Block',
+        77: 'Low Wood Block',
+    },
+    23: {  # Cuica
+        78: 'Mute Cuica',
+        79: 'Open Cuica',
+    },
+    24: {  # Triangles
+        80: 'Mute Triangle',
+        81: 'Open Triangle',
+    },
+}
+
+###################################################################################
+
+print('Module loaded!')
+print('=' * 70)
+print('Enjoy! :)')
+print('=' * 70)
+
 ###################################################################################
 # This is the end of the TMIDI X Python module
 ###################################################################################