pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32
 33    Returns
 34    -------
 35    json_string : str
 36        String for export to .json(.gz) file
 37    """
 38
 39    def _gen_data_d_from_list(ol):
 40        dl = []
 41        No = len(ol)
 42        for name in ol[0].mc_names:
 43            ed = {}
 44            ed['id'] = name
 45            ed['replica'] = []
 46            for r_name in ol[0].e_content[name]:
 47                rd = {}
 48                rd['name'] = r_name
 49                rd['deltas'] = []
 50                offsets = [o.r_values[r_name] - o.value for o in ol]
 51                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 52                for i in range(len(ol[0].idl[r_name])):
 53                    rd['deltas'].append([ol[0].idl[r_name][i]])
 54                    rd['deltas'][-1] += deltas[i].tolist()
 55                ed['replica'].append(rd)
 56            dl.append(ed)
 57        return dl
 58
 59    def _gen_cdata_d_from_list(ol):
 60        dl = []
 61        for name in ol[0].cov_names:
 62            ed = {}
 63            ed['id'] = name
 64            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 65            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 66            ncov = ol[0].covobs[name].cov.shape[0]
 67            ed['grad'] = []
 68            for i in range(ncov):
 69                ed['grad'].append([])
 70                for o in ol:
 71                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 72            dl.append(ed)
 73        return dl
 74
 75    def write_Obs_to_dict(o):
 76        d = {}
 77        d['type'] = 'Obs'
 78        d['layout'] = '1'
 79        if o.tag:
 80            d['tag'] = [o.tag]
 81        if o.reweighted:
 82            d['reweighted'] = o.reweighted
 83        d['value'] = [o.value]
 84        data = _gen_data_d_from_list([o])
 85        if len(data) > 0:
 86            d['data'] = data
 87        cdata = _gen_cdata_d_from_list([o])
 88        if len(cdata) > 0:
 89            d['cdata'] = cdata
 90        return d
 91
 92    def write_List_to_dict(ol):
 93        _assert_equal_properties(ol)
 94        d = {}
 95        d['type'] = 'List'
 96        d['layout'] = '%d' % len(ol)
 97        taglist = [o.tag for o in ol]
 98        if np.any([tag is not None for tag in taglist]):
 99            d['tag'] = taglist
100        if ol[0].reweighted:
101            d['reweighted'] = ol[0].reweighted
102        d['value'] = [o.value for o in ol]
103        data = _gen_data_d_from_list(ol)
104        if len(data) > 0:
105            d['data'] = data
106        cdata = _gen_cdata_d_from_list(ol)
107        if len(cdata) > 0:
108            d['cdata'] = cdata
109        return d
110
111    def write_Array_to_dict(oa):
112        ol = np.ravel(oa)
113        _assert_equal_properties(ol)
114        d = {}
115        d['type'] = 'Array'
116        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
117        taglist = [o.tag for o in ol]
118        if np.any([tag is not None for tag in taglist]):
119            d['tag'] = taglist
120        if ol[0].reweighted:
121            d['reweighted'] = ol[0].reweighted
122        d['value'] = [o.value for o in ol]
123        data = _gen_data_d_from_list(ol)
124        if len(data) > 0:
125            d['data'] = data
126        cdata = _gen_cdata_d_from_list(ol)
127        if len(cdata) > 0:
128            d['cdata'] = cdata
129        return d
130
131    def _nan_Obs_like(obs):
132        samples = []
133        names = []
134        idl = []
135        for key, value in obs.idl.items():
136            samples.append([np.nan] * len(value))
137            names.append(key)
138            idl.append(value)
139        my_obs = Obs(samples, names, idl)
140        my_obs._covobs = obs._covobs
141        for name in obs._covobs:
142            my_obs.names.append(name)
143        my_obs.reweighted = obs.reweighted
144        return my_obs
145
146    def write_Corr_to_dict(my_corr):
147        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
148        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
149        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
150        content = [o if o is not None else dummy_array for o in my_corr.content]
151        dat = write_Array_to_dict(np.array(content, dtype=object))
152        dat['type'] = 'Corr'
153        corr_meta_data = str(my_corr.tag)
154        if 'tag' in dat.keys():
155            dat['tag'].append(corr_meta_data)
156        else:
157            dat['tag'] = [corr_meta_data]
158        taglist = dat['tag']
159        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
160        dat['tag']['tag'] = taglist
161        if my_corr.prange is not None:
162            dat['tag']['prange'] = my_corr.prange
163        return dat
164
165    if not isinstance(ol, list):
166        ol = [ol]
167
168    d = {}
169    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
170    d['version'] = '1.1'
171    d['who'] = getpass.getuser()
172    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
173    d['host'] = socket.gethostname() + ', ' + platform.platform()
174
175    if description:
176        d['description'] = description
177
178    d['obsdata'] = []
179    for io in ol:
180        if isinstance(io, Obs):
181            d['obsdata'].append(write_Obs_to_dict(io))
182        elif isinstance(io, list):
183            d['obsdata'].append(write_List_to_dict(io))
184        elif isinstance(io, np.ndarray):
185            d['obsdata'].append(write_Array_to_dict(io))
186        elif isinstance(io, Corr):
187            d['obsdata'].append(write_Corr_to_dict(io))
188        else:
189            raise Exception("Unkown datatype.")
190
191    def _jsonifier(obj):
192        if isinstance(obj, dict):
193            result = {}
194            for key in obj:
195                if key is True:
196                    result['true'] = obj[key]
197                elif key is False:
198                    result['false'] = obj[key]
199                elif key is None:
200                    result['null'] = obj[key]
201                elif isinstance(key, (int, float, np.floating, np.integer)):
202                    result[str(key)] = obj[key]
203                else:
204                    raise TypeError('keys must be str, int, float, bool or None')
205            return result
206        elif isinstance(obj, np.integer):
207            return int(obj)
208        elif isinstance(obj, np.floating):
209            return float(obj)
210        else:
211            raise ValueError('%r is not JSON serializable' % (obj,))
212
213    if indent:
214        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
215    else:
216        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
217
218
219def dump_to_json(ol, fname, description='', indent=1, gz=True):
220    """Export a list of Obs or structures containing Obs to a .json(.gz) file.
221    Dict keys that are not JSON-serializable such as floats are converted to strings.
222
223    Parameters
224    ----------
225    ol : list
226        List of objects that will be exported. At the moment, these objects can be
227        either of: Obs, list, numpy.ndarray, Corr.
228        All Obs inside a structure have to be defined on the same set of configurations.
229    fname : str
230        Filename of the output file.
231    description : str
232        Optional string that describes the contents of the json file.
233    indent : int
234        Specify the indentation level of the json file. None or 0 is permissible and
235        saves disk space.
236    gz : bool
237        If True, the output is a gzipped json. If False, the output is a json file.
238
239    Returns
240    -------
241    Null
242    """
243
244    jsonstring = create_json_string(ol, description, indent)
245
246    if not fname.endswith('.json') and not fname.endswith('.gz'):
247        fname += '.json'
248
249    if gz:
250        if not fname.endswith('.gz'):
251            fname += '.gz'
252
253        fp = gzip.open(fname, 'wb')
254        fp.write(jsonstring.encode('utf-8'))
255    else:
256        fp = open(fname, 'w', encoding='utf-8')
257        fp.write(jsonstring)
258    fp.close()
259
260
261def _parse_json_dict(json_dict, verbose=True, full_output=False):
262    """Reconstruct a list of Obs or structures containing Obs from a dict that
263    was built out of a json string.
264
265    The following structures are supported: Obs, list, numpy.ndarray, Corr
266    If the list contains only one element, it is unpacked from the list.
267
268    Parameters
269    ----------
270    json_string : str
271        json string containing the data.
272    verbose : bool
273        Print additional information that was written to the file.
274    full_output : bool
275        If True, a dict containing auxiliary information and the data is returned.
276        If False, only the data is returned.
277
278    Returns
279    -------
280    result : list[Obs]
281        reconstructed list of observables from the json string
282    or
283    result : Obs
284        only one observable if the list only has one entry
285    or
286    result : dict
287        if full_output=True
288    """
289
290    def _gen_obsd_from_datad(d):
291        retd = {}
292        if d:
293            retd['names'] = []
294            retd['idl'] = []
295            retd['deltas'] = []
296            for ens in d:
297                for rep in ens['replica']:
298                    rep_name = rep['name']
299                    if len(rep_name) > len(ens["id"]):
300                        if rep_name[len(ens["id"])] != "|":
301                            tmp_list = list(rep_name)
302                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
303                            rep_name = ''.join(tmp_list)
304                    retd['names'].append(rep_name)
305                    retd['idl'].append([di[0] for di in rep['deltas']])
306                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
307        return retd
308
309    def _gen_covobsd_from_cdatad(d):
310        retd = {}
311        for ens in d:
312            retl = []
313            name = ens['id']
314            layouts = ens.get('layout', '1').strip()
315            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
316            cov = np.reshape(ens['cov'], layout)
317            grad = ens['grad']
318            nobs = len(grad[0])
319            for i in range(nobs):
320                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
321            retd[name] = retl
322        return retd
323
324    def get_Obs_from_dict(o):
325        layouts = o.get('layout', '1').strip()
326        if layouts != '1':
327            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
328
329        values = o['value']
330        od = _gen_obsd_from_datad(o.get('data', {}))
331        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
332
333        if od:
334            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
335            ret._value = values[0]
336        else:
337            ret = Obs([], [], means=[])
338            ret._value = values[0]
339        for name in cd:
340            co = cd[name][0]
341            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
342            ret.names.append(co['name'])
343
344        ret.reweighted = o.get('reweighted', False)
345        ret.tag = o.get('tag', [None])[0]
346        return ret
347
348    def get_List_from_dict(o):
349        layouts = o.get('layout', '1').strip()
350        layout = int(layouts)
351        values = o['value']
352        od = _gen_obsd_from_datad(o.get('data', {}))
353        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
354
355        ret = []
356        taglist = o.get('tag', layout * [None])
357        for i in range(layout):
358            if od:
359                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
360                ret[-1]._value = values[i]
361            else:
362                ret.append(Obs([], [], means=[]))
363                ret[-1]._value = values[i]
364                print('Created Obs with means= ', values[i])
365            for name in cd:
366                co = cd[name][i]
367                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
368                ret[-1].names.append(co['name'])
369
370            ret[-1].reweighted = o.get('reweighted', False)
371            ret[-1].tag = taglist[i]
372        return ret
373
374    def get_Array_from_dict(o):
375        layouts = o.get('layout', '1').strip()
376        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
377        N = np.prod(layout)
378        values = o['value']
379        od = _gen_obsd_from_datad(o.get('data', {}))
380        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
381
382        ret = []
383        taglist = o.get('tag', N * [None])
384        for i in range(N):
385            if od:
386                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
387                ret[-1]._value = values[i]
388            else:
389                ret.append(Obs([], [], means=[]))
390                ret[-1]._value = values[i]
391            for name in cd:
392                co = cd[name][i]
393                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
394                ret[-1].names.append(co['name'])
395            ret[-1].reweighted = o.get('reweighted', False)
396            ret[-1].tag = taglist[i]
397        return np.reshape(ret, layout)
398
399    def get_Corr_from_dict(o):
400        if isinstance(o.get('tag'), list):  # supports the old way
401            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
402            temp_prange = None
403        elif isinstance(o.get('tag'), dict):
404            tagdic = o.get('tag')
405            taglist = tagdic['tag']
406            if 'prange' in tagdic:
407                temp_prange = tagdic['prange']
408            else:
409                temp_prange = None
410        else:
411            raise Exception("The tag is not a list or dict")
412
413        corr_tag = taglist[-1]
414        tmp_o = o
415        tmp_o['tag'] = taglist[:-1]
416        if len(tmp_o['tag']) == 0:
417            del tmp_o['tag']
418        dat = get_Array_from_dict(tmp_o)
419        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
420        if corr_tag != 'None':
421            my_corr.tag = corr_tag
422
423        my_corr.prange = temp_prange
424        return my_corr
425
426    prog = json_dict.get('program', '')
427    version = json_dict.get('version', '')
428    who = json_dict.get('who', '')
429    date = json_dict.get('date', '')
430    host = json_dict.get('host', '')
431    if prog and verbose:
432        print('Data has been written using %s.' % (prog))
433    if version and verbose:
434        print('Format version %s' % (version))
435    if np.any([who, date, host] and verbose):
436        print('Written by %s on %s on host %s' % (who, date, host))
437    description = json_dict.get('description', '')
438    if description and verbose:
439        print()
440        print('Description: ', description)
441    obsdata = json_dict['obsdata']
442    ol = []
443    for io in obsdata:
444        if io['type'] == 'Obs':
445            ol.append(get_Obs_from_dict(io))
446        elif io['type'] == 'List':
447            ol.append(get_List_from_dict(io))
448        elif io['type'] == 'Array':
449            ol.append(get_Array_from_dict(io))
450        elif io['type'] == 'Corr':
451            ol.append(get_Corr_from_dict(io))
452        else:
453            raise Exception("Unknown datatype.")
454
455    if full_output:
456        retd = {}
457        retd['program'] = prog
458        retd['version'] = version
459        retd['who'] = who
460        retd['date'] = date
461        retd['host'] = host
462        retd['description'] = description
463        retd['obsdata'] = ol
464
465        return retd
466    else:
467        if len(obsdata) == 1:
468            ol = ol[0]
469
470        return ol
471
472
473def import_json_string(json_string, verbose=True, full_output=False):
474    """Reconstruct a list of Obs or structures containing Obs from a json string.
475
476    The following structures are supported: Obs, list, numpy.ndarray, Corr
477    If the list contains only one element, it is unpacked from the list.
478
479    Parameters
480    ----------
481    json_string : str
482        json string containing the data.
483    verbose : bool
484        Print additional information that was written to the file.
485    full_output : bool
486        If True, a dict containing auxiliary information and the data is returned.
487        If False, only the data is returned.
488
489    Returns
490    -------
491    result : list[Obs]
492        reconstructed list of observables from the json string
493    or
494    result : Obs
495        only one observable if the list only has one entry
496    or
497    result : dict
498        if full_output=True
499    """
500    return _parse_json_dict(json.loads(json_string), verbose, full_output)
501
502
503def load_json(fname, verbose=True, gz=True, full_output=False):
504    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
505
506    The following structures are supported: Obs, list, numpy.ndarray, Corr
507    If the list contains only one element, it is unpacked from the list.
508
509    Parameters
510    ----------
511    fname : str
512        Filename of the input file.
513    verbose : bool
514        Print additional information that was written to the file.
515    gz : bool
516        If True, assumes that data is gzipped. If False, assumes JSON file.
517    full_output : bool
518        If True, a dict containing auxiliary information and the data is returned.
519        If False, only the data is returned.
520
521    Returns
522    -------
523    result : list[Obs]
524        reconstructed list of observables from the json string
525    or
526    result : Obs
527        only one observable if the list only has one entry
528    or
529    result : dict
530        if full_output=True
531    """
532    if not fname.endswith('.json') and not fname.endswith('.gz'):
533        fname += '.json'
534    if gz:
535        if not fname.endswith('.gz'):
536            fname += '.gz'
537        with gzip.open(fname, 'r') as fin:
538            d = json.load(fin)
539    else:
540        if fname.endswith('.gz'):
541            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
542        with open(fname, 'r', encoding='utf-8') as fin:
543            d = json.loads(fin.read())
544
545    return _parse_json_dict(d, verbose, full_output)
546
547
548def _ol_from_dict(ind, reps='DICTOBS'):
549    """Convert a dictionary of Obs objects to a list and a dictionary that contains
550    placeholders instead of the Obs objects.
551
552    Parameters
553    ----------
554    ind : dict
555        Dict of JSON valid structures and objects that will be exported.
556        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
557        All Obs inside a structure have to be defined on the same set of configurations.
558    reps : str
559        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
560    """
561
562    obstypes = (Obs, Corr, np.ndarray)
563
564    if not reps.isalnum():
565        raise Exception('Placeholder string has to be alphanumeric!')
566    ol = []
567    counter = 0
568
569    def dict_replace_obs(d):
570        nonlocal ol
571        nonlocal counter
572        x = {}
573        for k, v in d.items():
574            if isinstance(v, dict):
575                v = dict_replace_obs(v)
576            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
577                v = obslist_replace_obs(v)
578            elif isinstance(v, list):
579                v = list_replace_obs(v)
580            elif isinstance(v, obstypes):
581                ol.append(v)
582                v = reps + '%d' % (counter)
583                counter += 1
584            elif isinstance(v, str):
585                if bool(re.match(r'%s[0-9]+' % (reps), v)):
586                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be safely exported.' % (v, reps))
587            x[k] = v
588        return x
589
590    def list_replace_obs(li):
591        nonlocal ol
592        nonlocal counter
593        x = []
594        for e in li:
595            if isinstance(e, list):
596                e = list_replace_obs(e)
597            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
598                e = obslist_replace_obs(e)
599            elif isinstance(e, dict):
600                e = dict_replace_obs(e)
601            elif isinstance(e, obstypes):
602                ol.append(e)
603                e = reps + '%d' % (counter)
604                counter += 1
605            elif isinstance(e, str):
606                if bool(re.match(r'%s[0-9]+' % (reps), e)):
607                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be safely exported.' % (e, reps))
608            x.append(e)
609        return x
610
611    def obslist_replace_obs(li):
612        nonlocal ol
613        nonlocal counter
614        il = []
615        for e in li:
616            il.append(e)
617
618        ol.append(il)
619        x = reps + '%d' % (counter)
620        counter += 1
621        return x
622
623    nd = dict_replace_obs(ind)
624
625    return ol, nd
626
627
628def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
629    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
630
631    Parameters
632    ----------
633    od : dict
634        Dict of JSON valid structures and objects that will be exported.
635        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
636        All Obs inside a structure have to be defined on the same set of configurations.
637    fname : str
638        Filename of the output file.
639    description : str
640        Optional string that describes the contents of the json file.
641    indent : int
642        Specify the indentation level of the json file. None or 0 is permissible and
643        saves disk space.
644    reps : str
645        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
646    gz : bool
647        If True, the output is a gzipped json. If False, the output is a json file.
648
649    Returns
650    -------
651    None
652    """
653
654    if not isinstance(od, dict):
655        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
656
657    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
658                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
659                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
660                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
661
662    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
663    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
664
665    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
666
667
668def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
669    """Parse a list of Obs or structures containing Obs and an accompanying
670    dict, where the structures have been replaced by placeholders to a
671    dict that contains the structures.
672
673    The following structures are supported: Obs, list, numpy.ndarray, Corr
674
675    Parameters
676    ----------
677    ol : list
678        List of objects -
679        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
680        All Obs inside a structure have to be defined on the same set of configurations.
681    ind : dict
682        Dict that defines the structure of the resulting dict and contains placeholders
683    reps : str
684        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
685    """
686    if not reps.isalnum():
687        raise Exception('Placeholder string has to be alphanumeric!')
688
689    counter = 0
690
691    def dict_replace_string(d):
692        nonlocal counter
693        nonlocal ol
694        x = {}
695        for k, v in d.items():
696            if isinstance(v, dict):
697                v = dict_replace_string(v)
698            elif isinstance(v, list):
699                v = list_replace_string(v)
700            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
701                index = int(v[len(reps):])
702                v = ol[index]
703                counter += 1
704            x[k] = v
705        return x
706
707    def list_replace_string(li):
708        nonlocal counter
709        nonlocal ol
710        x = []
711        for e in li:
712            if isinstance(e, list):
713                e = list_replace_string(e)
714            elif isinstance(e, dict):
715                e = dict_replace_string(e)
716            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
717                index = int(e[len(reps):])
718                e = ol[index]
719                counter += 1
720            x.append(e)
721        return x
722
723    nd = dict_replace_string(ind)
724
725    if counter == 0:
726        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
727
728    return nd
729
730
731def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
732    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
733
734    The following structures are supported: Obs, list, numpy.ndarray, Corr
735
736    Parameters
737    ----------
738    fname : str
739        Filename of the input file.
740    verbose : bool
741        Print additional information that was written to the file.
742    gz : bool
743        If True, assumes that data is gzipped. If False, assumes JSON file.
744    full_output : bool
745        If True, a dict containing auxiliary information and the data is returned.
746        If False, only the data is returned.
747    reps : str
748        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
749
750    Returns
751    -------
752    data : Obs / list / Corr
753        Read data
754    or
755    data : dict
756        Read data and meta-data
757    """
758    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
759    description = indata['description']['description']
760    indict = indata['description']['OBSDICT']
761    ol = indata['obsdata']
762    od = _od_from_list_and_dict(ol, indict, reps=reps)
763
764    if full_output:
765        indata['description'] = description
766        indata['obsdata'] = od
767        return indata
768    else:
769        return od