import copy
from datetime import datetime

import dask.array as da
import numpy as np
import pandas as pd
import pytest
import xarray as xr

from act.io.armfiles import read_netcdf
from act.qc.arm import add_dqr_to_qc
from act.qc.qcfilter import parse_bit, set_bit, unset_bit
from act.qc.radiometer_tests import fft_shading_test
from act.qc.sp2 import SP2ParticleCriteria, PYSP2_AVAILABLE
from act.tests import (
    EXAMPLE_CEIL1,
    EXAMPLE_CO2FLX4M,
    EXAMPLE_MET1,
    EXAMPLE_METE40,
    EXAMPLE_MFRSR,
    EXAMPLE_IRT25m20s,
    EXAMPLE_BRS,
)
from act.qc.bsrn_tests import _calculate_solar_parameters


def test_fft_shading_test():
    obj = read_netcdf(EXAMPLE_MFRSR)
    obj.clean.cleanup()
    obj = fft_shading_test(obj)
    qc_data = obj['qc_diffuse_hemisp_narrowband_filter4']
    assert np.nansum(qc_data.values) == 456


def test_global_qc_cleanup():
    ds_object = read_netcdf(EXAMPLE_MET1)
    ds_object.load()
    ds_object.clean.cleanup()

    assert ds_object['qc_wdir_vec_mean'].attrs['flag_meanings'] == [
        'Value is equal to missing_value.',
        'Value is less than the fail_min.',
        'Value is greater than the fail_max.',
    ]
    assert ds_object['qc_wdir_vec_mean'].attrs['flag_masks'] == [1, 2, 4]
    assert ds_object['qc_wdir_vec_mean'].attrs['flag_assessments'] == [
        'Bad',
        'Bad',
        'Bad',
    ]

    assert ds_object['qc_temp_mean'].attrs['flag_meanings'] == [
        'Value is equal to missing_value.',
        'Value is less than the fail_min.',
        'Value is greater than the fail_max.',
        'Difference between current and previous values exceeds fail_delta.',
    ]
    assert ds_object['qc_temp_mean'].attrs['flag_masks'] == [1, 2, 4, 8]
    assert ds_object['qc_temp_mean'].attrs['flag_assessments'] == [
        'Bad',
        'Bad',
        'Bad',
        'Indeterminate',
    ]

    ds_object.close()
    del ds_object


def test_qc_test_errors():
    ds_object = read_netcdf(EXAMPLE_MET1)
    var_name = 'temp_mean'

    assert ds_object.qcfilter.add_less_test(var_name, None) is None
    assert ds_object.qcfilter.add_greater_test(var_name, None) is None
    assert ds_object.qcfilter.add_less_equal_test(var_name, None) is None
    assert ds_object.qcfilter.add_equal_to_test(var_name, None) is None
    assert ds_object.qcfilter.add_not_equal_to_test(var_name, None) is None


def test_arm_qc():
    # Test DQR Webservice using known DQR
    variable = 'wspd_vec_mean'
    qc_variable = 'qc_' + variable
    obj = read_netcdf(EXAMPLE_METE40)

    # DQR webservice does go down, so ensure it
    # properly runs first before testing
    try:
        obj = add_dqr_to_qc(obj, variable=variable)
        ran = True
        obj.attrs['_datastream'] = obj.attrs['datastream']
        del obj.attrs['datastream']
        obj2 = add_dqr_to_qc(obj, variable=variable, add_qc_variable=qc_variable)
        obj3 = add_dqr_to_qc(obj)
        add_dqr_to_qc(obj, variable=variable, exclude=['D190529.4'])
        add_dqr_to_qc(obj, variable=variable, include=['D400101.1'])
        with np.testing.assert_raises(ValueError):
            del obj.attrs['_datastream']
            add_dqr_to_qc(obj, variable=variable)

    except ValueError:
        ran = False

    if ran:
        assert qc_variable in obj
        dqr = [True for d in obj[qc_variable].attrs['flag_meanings'] if 'D190529.4' in d]
        assert dqr[0] is True
        assert 'Suspect' not in obj[qc_variable].attrs['flag_assessments']
        assert 'Incorrect' not in obj[qc_variable].attrs['flag_assessments']

        assert qc_variable in obj2
        dqr = [True for d in obj2[qc_variable].attrs['flag_meanings'] if 'D190529.4' in d]
        assert dqr[0] is True
        assert 'Suspect' not in obj2[qc_variable].attrs['flag_assessments']
        assert 'Incorrect' not in obj2[qc_variable].attrs['flag_assessments']

        assert qc_variable in obj3
        dqr = [True for d in obj3[qc_variable].attrs['flag_meanings'] if 'D190529.4' in d]
        assert dqr[0] is True
        assert 'Suspect' not in obj3[qc_variable].attrs['flag_assessments']
        assert 'Incorrect' not in obj3[qc_variable].attrs['flag_assessments']


def test_qcfilter():
    ds_object = read_netcdf(EXAMPLE_IRT25m20s)
    var_name = 'inst_up_long_dome_resist'
    expected_qc_var_name = 'qc_' + var_name

    ds_object.qcfilter.check_for_ancillary_qc(
        var_name, add_if_missing=True, cleanup=False, flag_type=False
    )
    assert expected_qc_var_name in list(ds_object.keys())
    del ds_object[expected_qc_var_name]

    # Perform adding of quality control variables to object
    result = ds_object.qcfilter.add_test(var_name, test_meaning='Birds!')
    assert isinstance(result, dict)
    qc_var_name = result['qc_variable_name']
    assert qc_var_name == expected_qc_var_name

    # Check that new linking and describing attributes are set
    assert ds_object[qc_var_name].attrs['standard_name'] == 'quality_flag'
    assert ds_object[var_name].attrs['ancillary_variables'] == qc_var_name

    # Check that CF attributes are set including new flag_assessments
    assert 'flag_masks' in ds_object[qc_var_name].attrs.keys()
    assert 'flag_meanings' in ds_object[qc_var_name].attrs.keys()
    assert 'flag_assessments' in ds_object[qc_var_name].attrs.keys()

    # Check that the values of the attributes are set correctly
    assert ds_object[qc_var_name].attrs['flag_assessments'][0] == 'Bad'
    assert ds_object[qc_var_name].attrs['flag_meanings'][0] == 'Birds!'
    assert ds_object[qc_var_name].attrs['flag_masks'][0] == 1

    # Set some test values
    index = [0, 1, 2, 30]
    ds_object.qcfilter.set_test(var_name, index=index, test_number=result['test_number'])

    # Add a new test and set values
    index2 = [6, 7, 8, 50]
    ds_object.qcfilter.add_test(
        var_name,
        index=index2,
        test_number=9,
        test_meaning='testing high number',
        test_assessment='Suspect',
    )

    # Retrieve data from object as numpy masked array. Count number of masked
    # elements and ensure equal to size of index array.
    data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad')
    assert np.ma.count_masked(data) == len(index)

    data = ds_object.qcfilter.get_masked_data(
        var_name, rm_assessments='Suspect', return_nan_array=True
    )
    assert np.sum(np.isnan(data)) == len(index2)

    data = ds_object.qcfilter.get_masked_data(
        var_name, rm_assessments=['Bad', 'Suspect'], ma_fill_value=np.nan
    )
    assert np.ma.count_masked(data) == len(index + index2)

    # Test internal function for returning the index array of where the
    # tests are set.
    assert (
        np.sum(
            ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
            - np.array(index, dtype=int)
        )
        == 0
    )

    # Unset a test
    ds_object.qcfilter.unset_test(var_name, index=0, test_number=result['test_number'])
    # Remove the test
    ds_object.qcfilter.remove_test(var_name, test_number=33)
    ds_object.qcfilter.remove_test(var_name, test_number=result['test_number'])
    pytest.raises(ValueError, ds_object.qcfilter.add_test, var_name)
    pytest.raises(ValueError, ds_object.qcfilter.remove_test, var_name)

    ds_object.close()

    assert np.all(parse_bit([257]) == np.array([1, 9], dtype=np.int32))
    pytest.raises(ValueError, parse_bit, [1, 2])
    pytest.raises(ValueError, parse_bit, -1)

    assert set_bit(0, 16) == 32768
    data = range(0, 4)
    assert isinstance(set_bit(list(data), 2), list)
    assert isinstance(set_bit(tuple(data), 2), tuple)
    assert isinstance(unset_bit(list(data), 2), list)
    assert isinstance(unset_bit(tuple(data), 2), tuple)

    # Fill in missing tests
    ds_object = read_netcdf(EXAMPLE_IRT25m20s)
    del ds_object[var_name].attrs['long_name']
    # Test creating a qc variable
    ds_object.qcfilter.create_qc_variable(var_name)
    # Test creating a second qc variable and of flag type
    ds_object.qcfilter.create_qc_variable(var_name, flag_type=True)
    result = ds_object.qcfilter.add_test(
        var_name,
        index=[1, 2, 3],
        test_number=9,
        test_meaning='testing high number',
        flag_value=True,
    )
    ds_object.qcfilter.set_test(var_name, index=5, test_number=9, flag_value=True)
    data = ds_object.qcfilter.get_masked_data(var_name)
    assert np.isclose(np.sum(data), 42674.766, 0.01)
    data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad')
    assert np.isclose(np.sum(data), 42643.195, 0.01)

    ds_object.qcfilter.unset_test(var_name, test_number=9, flag_value=True)
    ds_object.qcfilter.unset_test(var_name, index=1, test_number=9, flag_value=True)
    assert ds_object.qcfilter.available_bit(result['qc_variable_name']) == 10
    assert ds_object.qcfilter.available_bit(result['qc_variable_name'], recycle=True) == 1
    ds_object.qcfilter.remove_test(var_name, test_number=9, flag_value=True)

    ds_object.qcfilter.update_ancillary_variable(var_name)
    # Test updating ancillary variable if does not exist
    ds_object.qcfilter.update_ancillary_variable('not_a_variable_name')
    # Change ancillary_variables attribute to test if add correct qc variable correctly
    ds_object[var_name].attrs['ancillary_variables'] = 'a_different_name'
    ds_object.qcfilter.update_ancillary_variable(var_name, qc_var_name=expected_qc_var_name)
    assert expected_qc_var_name in ds_object[var_name].attrs['ancillary_variables']

    # Test flag QC
    var_name = 'inst_sfc_ir_temp'
    qc_var_name = 'qc_' + var_name
    ds_object.qcfilter.create_qc_variable(var_name, flag_type=True)
    assert qc_var_name in list(ds_object.data_vars)
    assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
    assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
    del ds_object[qc_var_name]

    qc_var_name = ds_object.qcfilter.check_for_ancillary_qc(
        var_name, add_if_missing=True, cleanup=False, flag_type=True
    )
    assert qc_var_name in list(ds_object.data_vars)
    assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
    assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
    del ds_object[qc_var_name]

    ds_object.qcfilter.add_missing_value_test(var_name, flag_value=True, prepend_text='arm')
    ds_object.qcfilter.add_test(
        var_name,
        index=list(range(0, 20)),
        test_number=2,
        test_meaning='Testing flag',
        flag_value=True,
        test_assessment='Suspect',
    )
    assert qc_var_name in list(ds_object.data_vars)
    assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
    assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
    assert 'standard_name' in ds_object[qc_var_name].attrs.keys()
    assert ds_object[qc_var_name].attrs['flag_values'] == [1, 2]
    assert ds_object[qc_var_name].attrs['flag_assessments'] == ['Bad', 'Suspect']

    ds_object.close()


def test_qcfilter2():
    ds_object = read_netcdf(EXAMPLE_IRT25m20s)
    var_name = 'inst_up_long_dome_resist'
    expected_qc_var_name = 'qc_' + var_name

    data = ds_object[var_name].values
    data[0:4] = data[0:4] + 30.0
    data[1000:1024] = data[1000:1024] + 30.0
    ds_object[var_name].values = data

    coef = 1.4
    ds_object.qcfilter.add_iqr_test(var_name, coef=1.4, test_assessment='Bad', prepend_text='arm')
    assert np.sum(ds_object[expected_qc_var_name].values) == 28
    assert ds_object[expected_qc_var_name].attrs['flag_masks'] == [1]
    assert ds_object[expected_qc_var_name].attrs['flag_meanings'] == [
        f'arm: Value outside of interquartile range test range with a coefficient of {coef}'
    ]

    ds_object.qcfilter.add_iqr_test(var_name, test_number=3, prepend_text='ACT')
    assert np.sum(ds_object[expected_qc_var_name].values) == 140
    assert ds_object[expected_qc_var_name].attrs['flag_masks'] == [1, 4]
    assert ds_object[expected_qc_var_name].attrs['flag_meanings'][-1] == (
        'ACT: Value outside of interquartile range test range with a coefficient of 1.5'
    )

    ds_object.qcfilter.add_gesd_test(var_name, test_assessment='Bad')
    assert np.sum(ds_object[expected_qc_var_name].values) == 204
    assert ds_object[expected_qc_var_name].attrs['flag_masks'] == [1, 4, 8]
    assert ds_object[expected_qc_var_name].attrs['flag_meanings'][-1] == (
        'Value failed generalized Extreme Studentized Deviate test with an alpha of 0.05'
    )

    ds_object.qcfilter.add_gesd_test(var_name, alpha=0.1)
    assert np.sum(ds_object[expected_qc_var_name].values) == 332
    assert ds_object[expected_qc_var_name].attrs['flag_masks'] == [1, 4, 8, 16]
    assert ds_object[expected_qc_var_name].attrs['flag_meanings'][-1] == (
        'Value failed generalized Extreme Studentized Deviate test with an alpha of 0.1'
    )
    assert ds_object[expected_qc_var_name].attrs['flag_assessments'] == [
        'Bad',
        'Indeterminate',
        'Bad',
        'Indeterminate',
    ]


def test_qcfilter3():
    ds_object = read_netcdf(EXAMPLE_IRT25m20s)
    var_name = 'inst_up_long_dome_resist'
    result = ds_object.qcfilter.add_test(var_name, index=range(0, 100), test_meaning='testing')
    qc_var_name = result['qc_variable_name']
    assert ds_object[qc_var_name].values.dtype.kind in np.typecodes['AllInteger']

    ds_object[qc_var_name].values = ds_object[qc_var_name].values.astype(np.float32)
    assert ds_object[qc_var_name].values.dtype.kind not in np.typecodes['AllInteger']

    result = ds_object.qcfilter.get_qc_test_mask(
        var_name=var_name, test_number=1, return_index=False
    )
    assert np.sum(result) == 100
    result = ds_object.qcfilter.get_qc_test_mask(
        var_name=var_name, test_number=1, return_index=True
    )
    assert np.sum(result) == 4950

    # Test where QC variables are not integer type
    ds_object = ds_object.resample(time='5min').mean(keep_attrs=True)
    ds_object.qcfilter.add_test(
        var_name, index=range(0, ds_object.time.size), test_meaning='Testing float'
    )
    assert np.sum(ds_object[qc_var_name].values) == 582

    ds_object[qc_var_name].values = ds_object[qc_var_name].values.astype(np.float32)
    ds_object.qcfilter.remove_test(var_name, test_number=2)
    assert np.sum(ds_object[qc_var_name].values) == 6


def test_qctests():
    ds_object = read_netcdf(EXAMPLE_IRT25m20s)
    var_name = 'inst_up_long_dome_resist'

    # Add in one missing value and test for that missing value
    data = ds_object[var_name].values
    data[0] = np.nan
    ds_object[var_name].data = da.from_array(data)
    result = ds_object.qcfilter.add_missing_value_test(var_name)
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert data.mask[0]

    result = ds_object.qcfilter.add_missing_value_test(var_name, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert data == np.array([0])
    ds_object.qcfilter.remove_test(var_name, test_number=result['test_number'])

    # less than min test
    limit_value = 6.8
    result = ds_object.qcfilter.add_less_test(
        var_name, limit_value, prepend_text='arm', limit_attr_name='fail_min'
    )

    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 54
    assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_min'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_min'], limit_value)

    result = ds_object.qcfilter.add_less_test(var_name, limit_value, test_assessment='Suspect')
    assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys()

    limit_value = 8
    result = ds_object.qcfilter.add_less_test(var_name, limit_value)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 2911939
    result = ds_object.qcfilter.add_less_test(var_name, limit_value, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 2911939

    # greator than max test
    limit_value = 12.7
    result = ds_object.qcfilter.add_greater_test(
        var_name, limit_value, prepend_text='arm', limit_attr_name='fail_max'
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 61
    assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_max'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_max'], limit_value)

    result = ds_object.qcfilter.add_greater_test(var_name, limit_value, test_assessment='Suspect')
    assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_greater_test(var_name, limit_value, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 125458
    result = ds_object.qcfilter.add_greater_test(var_name, limit_value)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 125458

    # less than or equal test
    limit_value = 6.9
    result = ds_object.qcfilter.add_less_equal_test(
        var_name,
        limit_value,
        test_assessment='Suspect',
        prepend_text='arm',
        limit_attr_name='warn_min',
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 149
    assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['warn_min'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_min'], limit_value)

    result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value)
    assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 601581
    result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 601581

    # greater than or equal test
    result = ds_object.qcfilter.add_greater_equal_test(var_name, None)
    limit_value = 12
    result = ds_object.qcfilter.add_greater_equal_test(
        var_name,
        limit_value,
        test_assessment='Suspect',
        prepend_text='arm',
        limit_attr_name='warn_max',
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 606
    assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['warn_max'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_max'], limit_value)

    result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value)
    assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 1189873
    result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 1189873

    # equal to test
    limit_value = 7.6705
    result = ds_object.qcfilter.add_equal_to_test(
        var_name, limit_value, prepend_text='arm', limit_attr_name='fail_equal_to'
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 2
    assert 'fail_equal_to' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_equal_to'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_equal_to'], limit_value)

    result = ds_object.qcfilter.add_equal_to_test(
        var_name, limit_value, test_assessment='Indeterminate'
    )
    assert 'warn_equal_to' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_equal_to_test(var_name, limit_value, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 8631
    result = ds_object.qcfilter.add_equal_to_test(var_name, limit_value)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 8631

    # not equal to test
    limit_value = 7.6705
    result = ds_object.qcfilter.add_not_equal_to_test(
        var_name,
        limit_value,
        test_assessment='Indeterminate',
        prepend_text='arm',
        limit_attr_name='warn_not_equal_to',
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 4318
    assert 'warn_not_equal_to' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'], limit_value)

    result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value)
    assert 'fail_not_equal_to' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 9320409
    result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 9320409

    # outside range test
    limit_value1 = 6.8
    limit_value2 = 12.7
    result = ds_object.qcfilter.add_outside_test(
        var_name,
        limit_value1,
        limit_value2,
        prepend_text='arm',
        limit_attr_names=['fail_lower_range', 'fail_upper_range'],
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 115
    assert 'fail_lower_range' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_lower_range'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_lower_range'], limit_value1)
    assert 'fail_upper_range' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_upper_range'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_upper_range'], limit_value2)

    result = ds_object.qcfilter.add_outside_test(
        var_name, limit_value1, limit_value2, test_assessment='Indeterminate'
    )
    assert 'warn_lower_range' in ds_object[result['qc_variable_name']].attrs.keys()
    assert 'warn_upper_range' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_outside_test(
        var_name, limit_value1, limit_value2, use_dask=True
    )
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 342254
    result = ds_object.qcfilter.add_outside_test(
        var_name,
        limit_value1,
        limit_value2,
    )
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 342254

    # Starting to run out of space for tests. Remove some tests.
    for ii in range(16, 30):
        ds_object.qcfilter.remove_test(var_name, test_number=ii)

    # inside range test
    limit_value1 = 7
    limit_value2 = 8
    result = ds_object.qcfilter.add_inside_test(
        var_name,
        limit_value1,
        limit_value2,
        prepend_text='arm',
        limit_attr_names=['fail_lower_range_inner', 'fail_upper_range_inner'],
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 479
    assert 'fail_lower_range_inner' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(
        ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'],
        limit_value1,
    )
    assert 'fail_upper_range_inner' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(
        ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'],
        limit_value2,
    )

    result = ds_object.qcfilter.add_inside_test(
        var_name, limit_value1, limit_value2, test_assessment='Indeterminate'
    )
    assert 'warn_lower_range_inner' in ds_object[result['qc_variable_name']].attrs.keys()
    assert 'warn_upper_range_inner' in ds_object[result['qc_variable_name']].attrs.keys()

    result = ds_object.qcfilter.add_inside_test(var_name, limit_value1, limit_value2, use_dask=True)
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 1820693
    result = ds_object.qcfilter.add_inside_test(
        var_name,
        limit_value1,
        limit_value2,
    )
    data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True)
    assert np.sum(data) == 1820693

    # delta test
    test_limit = 0.05
    result = ds_object.qcfilter.add_delta_test(
        var_name, test_limit, prepend_text='arm', limit_attr_name='warn_delta'
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert np.ma.count_masked(data) == 175
    assert 'warn_delta' in ds_object[result['qc_variable_name']].attrs.keys()
    assert (
        ds_object[result['qc_variable_name']].attrs['warn_delta'].dtype
        == ds_object[result['variable_name']].values.dtype
    )
    assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_delta'], test_limit)

    data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments=['Suspect', 'Bad'])
    assert np.ma.count_masked(data) == 1355

    result = ds_object.qcfilter.add_delta_test(var_name, test_limit, test_assessment='Bad')
    assert 'fail_delta' in ds_object[result['qc_variable_name']].attrs.keys()

    comp_object = read_netcdf(EXAMPLE_IRT25m20s)
    with np.testing.assert_raises(ValueError):
        result = ds_object.qcfilter.add_difference_test(var_name, 'test')

    with np.testing.assert_raises(ValueError):
        result = ds_object.qcfilter.add_difference_test(
            var_name,
            {comp_object.attrs['datastream']: comp_object},
            var_name,
            diff_limit=None,
        )

    assert ds_object.qcfilter.add_difference_test(var_name, set_test_regardless=False) is None

    result = ds_object.qcfilter.add_difference_test(
        var_name,
        {comp_object.attrs['datastream']: comp_object},
        var_name,
        diff_limit=1,
        prepend_text='arm',
    )
    data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number'])
    assert 'arm' in result['test_meaning']
    assert not (data.mask).all()

    comp_object.close()
    ds_object.close()


def test_qctests_dos():
    ds_object = read_netcdf(EXAMPLE_IRT25m20s)
    var_name = 'inst_up_long_dome_resist'

    # persistence test
    data = ds_object[var_name].values
    data[1000:2500] = data[1000]
    ds_object[var_name].values = data
    result = ds_object.qcfilter.add_persistence_test(var_name)
    qc_var_name = result['qc_variable_name']
    test_meaning = (
        'Data failing persistence test. Standard Deviation over a '
        'window of 10 values less than 0.0001.'
    )
    assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
    # There is a precision issue with GitHub testing that makes the number of tests
    # tripped off by 1. This isclose() option is to account for that.
    assert np.isclose(np.sum(ds_object[qc_var_name].values), 1500, atol=1)

    ds_object.qcfilter.add_persistence_test(var_name, window=10000, prepend_text='DQO')
    test_meaning = (
        'DQO: Data failing persistence test. Standard Deviation over a window of '
        '4320 values less than 0.0001.'
    )
    assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning


def test_datafilter():
    ds = read_netcdf(EXAMPLE_MET1, drop_variables=['base_time', 'time_offset'])
    ds.clean.cleanup()

    data_var_names = list(ds.data_vars)
    qc_var_names = [var_name for var_name in ds.data_vars if var_name.startswith('qc_')]
    data_var_names = list(set(data_var_names) - set(qc_var_names))
    data_var_names.sort()
    qc_var_names.sort()

    var_name = 'atmos_pressure'

    ds_1 = ds.mean()

    ds.qcfilter.add_less_test(var_name, 99, test_assessment='Bad')
    ds_filtered = copy.deepcopy(ds)
    ds_filtered.qcfilter.datafilter(rm_assessments='Bad', del_qc_var=False)
    ds_2 = ds_filtered.mean()
    assert np.isclose(ds_1[var_name].values, 98.86, atol=0.01)
    assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01)
    assert isinstance(ds_1[var_name].data, da.core.Array)

    ds_filtered = copy.deepcopy(ds)
    ds_filtered.qcfilter.datafilter(rm_assessments='Bad', variables=var_name)
    ds_2 = ds_filtered.mean()
    assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01)
    expected_var_names = sorted(list(set(data_var_names + qc_var_names) - set(['qc_' + var_name])))
    assert sorted(list(ds_filtered.data_vars)) == expected_var_names

    ds_filtered = copy.deepcopy(ds)
    ds_filtered.qcfilter.datafilter(rm_assessments='Bad', del_qc_var=True)
    assert sorted(list(ds_filtered.data_vars)) == data_var_names

    ds.close()
    del ds


def test_qc_remainder():
    ds = read_netcdf(EXAMPLE_MET1)
    assert ds.clean.get_attr_info(variable='bad_name') is None
    del ds.attrs['qc_bit_comment']
    assert isinstance(ds.clean.get_attr_info(), dict)
    ds.attrs['qc_flag_comment'] = 'testing'
    ds.close()

    ds = read_netcdf(EXAMPLE_MET1)
    ds.clean.cleanup(normalize_assessment=True)
    ds['qc_atmos_pressure'].attrs['units'] = 'testing'
    del ds['qc_temp_mean'].attrs['units']
    del ds['qc_temp_mean'].attrs['flag_masks']
    ds.clean.handle_missing_values()
    ds.close()

    ds = read_netcdf(EXAMPLE_MET1)
    ds.attrs['qc_bit_1_comment'] = 'tesing'
    data = ds['qc_atmos_pressure'].values.astype(np.int64)
    data[0] = 2**32
    ds['qc_atmos_pressure'].values = data
    ds.clean.get_attr_info(variable='qc_atmos_pressure')
    ds.clean.clean_arm_state_variables('testname')
    ds.clean.cleanup()
    ds['qc_atmos_pressure'].attrs['standard_name'] = 'wrong_name'
    ds.clean.link_variables()
    assert ds['qc_atmos_pressure'].attrs['standard_name'] == 'quality_flag'
    ds.close()


def test_qc_flag_description():
    """
    This will check if the cleanup() method will correctly convert convert
    flag_#_description to CF flag_masks and flag_meanings.

    """

    ds = read_netcdf(EXAMPLE_CO2FLX4M)
    ds.clean.cleanup()
    qc_var_name = ds.qcfilter.check_for_ancillary_qc(
        'momentum_flux', add_if_missing=False, cleanup=False
    )

    assert isinstance(ds[qc_var_name].attrs['flag_masks'], list)
    assert isinstance(ds[qc_var_name].attrs['flag_meanings'], list)
    assert isinstance(ds[qc_var_name].attrs['flag_assessments'], list)
    assert ds[qc_var_name].attrs['standard_name'] == 'quality_flag'

    assert len(ds[qc_var_name].attrs['flag_masks']) == 9
    unique_flag_assessments = list({'Acceptable', 'Indeterminate', 'Bad'})
    assert list(set(ds[qc_var_name].attrs['flag_assessments'])) == unique_flag_assessments


def test_clean():
    # Read test data
    ceil_ds = read_netcdf([EXAMPLE_CEIL1])
    # Cleanup QC data
    ceil_ds.clean.cleanup(clean_arm_state_vars=['detection_status'])

    # Check that global attribures are removed
    global_attributes = [
        'qc_bit_comment',
        'qc_bit_1_description',
        'qc_bit_1_assessment',
        'qc_bit_2_description',
        'qc_bit_2_assessment' 'qc_bit_3_description',
        'qc_bit_3_assessment',
    ]

    for glb_att in global_attributes:
        assert glb_att not in ceil_ds.attrs.keys()

    # Check that CF attributes are set including new flag_assessments
    var_name = 'qc_first_cbh'
    for attr_name in ['flag_masks', 'flag_meanings', 'flag_assessments']:
        assert attr_name in ceil_ds[var_name].attrs.keys()
        assert isinstance(ceil_ds[var_name].attrs[attr_name], list)

    # Check that the flag_mask values are set correctly
    assert ceil_ds['qc_first_cbh'].attrs['flag_masks'] == [1, 2, 4]

    # Check that the flag_meanings values are set correctly
    assert ceil_ds['qc_first_cbh'].attrs['flag_meanings'] == [
        'Value is equal to missing_value.',
        'Value is less than the fail_min.',
        'Value is greater than the fail_max.',
    ]

    # Check the value of flag_assessments is as expected
    assert ceil_ds['qc_first_cbh'].attrs['flag_assessments'] == ['Bad', 'Bad', 'Bad']

    # Check that ancillary varibles is being added
    assert 'qc_first_cbh' in ceil_ds['first_cbh'].attrs['ancillary_variables'].split()

    # Check that state field is updated to CF
    assert 'flag_values' in ceil_ds['detection_status'].attrs.keys()
    assert isinstance(ceil_ds['detection_status'].attrs['flag_values'], list)
    assert ceil_ds['detection_status'].attrs['flag_values'] == [0, 1, 2, 3, 4, 5]

    assert 'flag_meanings' in ceil_ds['detection_status'].attrs.keys()
    assert isinstance(ceil_ds['detection_status'].attrs['flag_meanings'], list)
    assert ceil_ds['detection_status'].attrs['flag_meanings'] == [
        'No significant backscatter',
        'One cloud base detected',
        'Two cloud bases detected',
        'Three cloud bases detected',
        'Full obscuration determined but no cloud base detected',
        'Some obscuration detected but determined to be transparent',
    ]

    assert 'flag_0_description' not in ceil_ds['detection_status'].attrs.keys()
    assert 'detection_status' in ceil_ds['first_cbh'].attrs['ancillary_variables'].split()

    ceil_ds.close()


def test_compare_time_series_trends():

    drop_vars = [
        'base_time',
        'time_offset',
        'atmos_pressure',
        'qc_atmos_pressure',
        'temp_std',
        'rh_mean',
        'qc_rh_mean',
        'rh_std',
        'vapor_pressure_mean',
        'qc_vapor_pressure_mean',
        'vapor_pressure_std',
        'wspd_arith_mean',
        'qc_wspd_arith_mean',
        'wspd_vec_mean',
        'qc_wspd_vec_mean',
        'wdir_vec_mean',
        'qc_wdir_vec_mean',
        'wdir_vec_std',
        'tbrg_precip_total',
        'qc_tbrg_precip_total',
        'tbrg_precip_total_corr',
        'qc_tbrg_precip_total_corr',
        'org_precip_rate_mean',
        'qc_org_precip_rate_mean',
        'pwd_err_code',
        'pwd_mean_vis_1min',
        'qc_pwd_mean_vis_1min',
        'pwd_mean_vis_10min',
        'qc_pwd_mean_vis_10min',
        'pwd_pw_code_inst',
        'qc_pwd_pw_code_inst',
        'pwd_pw_code_15min',
        'qc_pwd_pw_code_15min',
        'pwd_pw_code_1hr',
        'qc_pwd_pw_code_1hr',
        'pwd_precip_rate_mean_1min',
        'qc_pwd_precip_rate_mean_1min',
        'pwd_cumul_rain',
        'qc_pwd_cumul_rain',
        'pwd_cumul_snow',
        'qc_pwd_cumul_snow',
        'logger_volt',
        'qc_logger_volt',
        'logger_temp',
        'qc_logger_temp',
        'lat',
        'lon',
        'alt',
    ]
    ds = read_netcdf(EXAMPLE_MET1, drop_variables=drop_vars)
    ds.clean.cleanup()
    ds2 = copy.deepcopy(ds)

    var_name = 'temp_mean'
    qc_var_name = ds.qcfilter.check_for_ancillary_qc(
        var_name, add_if_missing=False, cleanup=False, flag_type=False
    )
    ds.qcfilter.compare_time_series_trends(
        var_name=var_name,
        time_shift=60,
        comp_var_name=var_name,
        comp_dataset=ds2,
        time_qc_threshold=60 * 10,
    )

    test_description = (
        'Time shift detected with Minimum Difference test. Comparison of '
        'temp_mean with temp_mean off by 0 seconds exceeding absolute '
        'threshold of 600 seconds.'
    )
    assert ds[qc_var_name].attrs['flag_meanings'][-1] == test_description

    time = ds2['time'].values + np.timedelta64(1, 'h')
    time_attrs = ds2['time'].attrs
    ds2 = ds2.assign_coords({'time': time})
    ds2['time'].attrs = time_attrs

    ds.qcfilter.compare_time_series_trends(
        var_name=var_name, comp_dataset=ds2, time_step=60, time_match_threshhold=50
    )

    test_description = (
        'Time shift detected with Minimum Difference test. Comparison of '
        'temp_mean with temp_mean off by 3600 seconds exceeding absolute '
        'threshold of 900 seconds.'
    )
    assert ds[qc_var_name].attrs['flag_meanings'][-1] == test_description


def test_qc_data_type():
    drop_vars = [
        'base_time',
        'time_offset',
        'inst_up_long_case_resist',
        'inst_up_long_hemisp_tp',
        'inst_up_short_hemisp_tp',
        'inst_sfc_ir_temp',
        'lat',
        'lon',
        'alt',
    ]
    ds_object = read_netcdf(EXAMPLE_IRT25m20s, drop_variables=drop_vars)
    var_name = 'inst_up_long_dome_resist'
    expected_qc_var_name = 'qc_' + var_name
    ds_object.qcfilter.check_for_ancillary_qc(var_name, add_if_missing=True)
    del ds_object[expected_qc_var_name].attrs['flag_meanings']
    del ds_object[expected_qc_var_name].attrs['flag_assessments']
    ds_object[expected_qc_var_name] = ds_object[expected_qc_var_name].astype(np.int8)
    ds_object.qcfilter.add_test(var_name, index=[1], test_number=9, test_meaning='First test')

    assert ds_object[expected_qc_var_name].attrs['flag_masks'][0].dtype == np.uint32
    assert ds_object[expected_qc_var_name].dtype == np.int16
    ds_object.qcfilter.add_test(var_name, index=[1], test_number=17, test_meaning='Second test')
    assert ds_object[expected_qc_var_name].dtype == np.int32
    ds_object.qcfilter.add_test(var_name, index=[1], test_number=33, test_meaning='Third test')
    assert ds_object[expected_qc_var_name].dtype == np.int64
    assert ds_object[expected_qc_var_name].attrs['flag_masks'][0].dtype == np.uint64

    ds_object.qcfilter.add_test(var_name, index=[1], test_meaning='Fourth test', recycle=True)


def test_qc_speed():
    """
    This tests the speed of the QC module to ensure changes do not significantly
    slow down the module's processing.
    """

    n_variables = 100
    n_samples = 100

    time = pd.date_range(start='2022-02-17 00:00:00', end='2022-02-18 00:00:00', periods=n_samples)

    # Create data variables with random noise
    np.random.seed(42)
    noisy_data_mapping = {f'data_var_{i}': np.random.random(time.shape) for i in range(n_variables)}

    ds = xr.Dataset(
        data_vars={name: ('time', data) for name, data in noisy_data_mapping.items()},
        coords={'time': time},
    )

    start = datetime.utcnow()
    for name, var in noisy_data_mapping.items():
        failed_qc = var > 0.75  # Consider data above 0.75 as bad. Negligible time here.
        ds.qcfilter.add_test(name, index=failed_qc, test_meaning='Value above threshold')

    time_diff = datetime.utcnow() - start
    assert time_diff.seconds <= 3


@pytest.mark.skipif(not PYSP2_AVAILABLE, reason="PySP2 is not installed.")
def test_sp2_particle_config():
    particle_config_object = SP2ParticleCriteria()
    assert particle_config_object.ScatMaxPeakHt1 == 60000
    assert particle_config_object.ScatMinPeakHt1 == 250
    assert particle_config_object.ScatMaxPeakHt2 == 60000
    assert particle_config_object.ScatMinPeakHt2 == 250
    assert particle_config_object.ScatMinWidth == 10
    assert particle_config_object.ScatMaxWidth == 90
    assert particle_config_object.ScatMinPeakPos == 20
    assert particle_config_object.ScatMaxPeakPos == 90
    assert particle_config_object.IncanMinPeakHt1 == 200
    assert particle_config_object.IncanMinPeakHt2 == 200
    assert particle_config_object.IncanMaxPeakHt1 == 60000
    assert particle_config_object.IncanMaxPeakHt2 == 60000
    assert particle_config_object.IncanMinWidth == 5
    assert particle_config_object.IncanMaxWidth == np.inf
    assert particle_config_object.IncanMinPeakPos == 20
    assert particle_config_object.IncanMaxPeakPos == 90
    assert particle_config_object.IncanMinPeakRatio == 0.1
    assert particle_config_object.IncanMaxPeakRatio == 25
    assert particle_config_object.IncanMaxPeakOffset == 11
    assert particle_config_object.c0Mass1 == 0
    assert particle_config_object.c1Mass1 == 0.0001896
    assert particle_config_object.c2Mass1 == 0
    assert particle_config_object.c3Mass1 == 0
    assert particle_config_object.c0Mass2 == 0
    assert particle_config_object.c1Mass2 == 0.0016815
    assert particle_config_object.c2Mass2 == 0
    assert particle_config_object.c3Mass2 == 0
    assert particle_config_object.c0Scat1 == 0
    assert particle_config_object.c1Scat1 == 78.141
    assert particle_config_object.c2Scat1 == 0
    assert particle_config_object.c0Scat2 == 0
    assert particle_config_object.c1Scat2 == 752.53
    assert particle_config_object.c2Scat2 == 0
    assert particle_config_object.densitySO4 == 1.8
    assert particle_config_object.densityBC == 1.8
    assert particle_config_object.TempSTP == 273.15
    assert particle_config_object.PressSTP == 1013.25


def test_bsrn_limits_test():

    for use_dask in [False, True]:
        ds_object = read_netcdf(EXAMPLE_BRS)
        var_names = list(ds_object.data_vars)
        # Remove QC variables to make testing easier
        for var_name in var_names:
            if var_name.startswith('qc_'):
                del ds_object[var_name]

        # Add atmospheric temperature fake data
        ds_object['temp_mean'] = xr.DataArray(
            data=np.full(ds_object.time.size, 13.5), dims=['time'],
            attrs={'long_name': 'Atmospheric air temperature', 'units': 'degC'})

        # Make a short direct variable since BRS does not have one
        ds_object['short_direct'] = copy.deepcopy(ds_object['short_direct_normal'])
        ds_object['short_direct'].attrs['ancillary_variables'] = 'qc_short_direct'
        ds_object['short_direct'].attrs['long_name'] = 'Shortwave direct irradiance, pyrheliometer'
        sza, Sa = _calculate_solar_parameters(ds_object, 'lat', 'lon', 1360.8)
        ds_object['short_direct'].data = ds_object['short_direct'].data * .5

        # Make up long variable since BRS does not have values
        ds_object['up_long_hemisp'].data = copy.deepcopy(ds_object['down_long_hemisp_shaded'].data)
        data = copy.deepcopy(ds_object['down_short_hemisp'].data)
        ds_object['up_short_hemisp'].data = data

        # Test that nothing happens when no variable names are provided
        ds_object.qcfilter.bsrn_limits_test()

        # Mess with data to get tests to trip
        data = ds_object['down_short_hemisp'].data
        data[200:300] = data[200:300] - 10
        data[800:850] += 330
        data[1340:1380] += 600
        ds_object['down_short_hemisp'].data = data

        data = ds_object['down_short_diffuse_hemisp'].data
        data[200:250] = data[200:250] - 1.9
        data[250:300] = data[250:300] - 3.9
        data[800:850] += 330
        data[1340:1380] += 600
        ds_object['down_short_diffuse_hemisp'].data = data

        data = ds_object['short_direct_normal'].data
        data[200:250] = data[200:250] - 1.9
        data[250:300] = data[250:300] - 3.9
        data[800:850] += 600
        data[1340:1380] += 800
        ds_object['short_direct_normal'].data = data

        data = ds_object['short_direct'].data
        data[200:250] = data[200:250] - 1.9
        data[250:300] = data[250:300] - 3.9
        data[800:850] += 300
        data[1340:1380] += 800
        ds_object['short_direct'].data = data

        data = ds_object['down_long_hemisp_shaded'].data
        data[200:250] = data[200:250] - 355
        data[250:300] = data[250:300] - 400
        data[800:850] += 200
        data[1340:1380] += 400
        ds_object['down_long_hemisp_shaded'].data = data

        data = ds_object['up_long_hemisp'].data
        data[200:250] = data[200:250] - 355
        data[250:300] = data[250:300] - 400
        data[800:850] += 300
        data[1340:1380] += 500
        ds_object['up_long_hemisp'].data = data

        ds_object.qcfilter.bsrn_limits_test(
            gbl_SW_dn_name='down_short_hemisp',
            glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',
            direct_normal_SW_dn_name='short_direct_normal',
            glb_SW_up_name='up_short_hemisp',
            glb_LW_dn_name='down_long_hemisp_shaded',
            glb_LW_up_name='up_long_hemisp',
            direct_SW_dn_name='short_direct',
            use_dask=use_dask)

        assert ds_object['qc_down_short_hemisp'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_down_short_hemisp'].attrs['flag_meanings'][-2] == \
            'Value less than BSRN physically possible limit of -4.0 W/m^2'
        assert ds_object['qc_down_short_hemisp'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN physically possible limit'

        assert ds_object['qc_down_short_diffuse_hemisp'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_down_short_diffuse_hemisp'].attrs['flag_assessments'] == ['Bad', 'Bad']

        assert ds_object['qc_short_direct'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_short_direct'].attrs['flag_assessments'] == ['Bad', 'Bad']
        assert ds_object['qc_short_direct'].attrs['flag_meanings'] == \
            ['Value less than BSRN physically possible limit of -4.0 W/m^2',
             'Value greater than BSRN physically possible limit']

        assert ds_object['qc_short_direct_normal'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_short_direct_normal'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN physically possible limit'

        assert ds_object['qc_down_short_hemisp'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_down_short_hemisp'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN physically possible limit'

        assert ds_object['qc_up_short_hemisp'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_up_short_hemisp'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN physically possible limit'

        assert ds_object['qc_up_long_hemisp'].attrs['flag_masks'] == [1, 2]
        assert ds_object['qc_up_long_hemisp'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN physically possible limit of 900.0 W/m^2'

        ds_object.qcfilter.bsrn_limits_test(
            test="Extremely Rare",
            gbl_SW_dn_name='down_short_hemisp',
            glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',
            direct_normal_SW_dn_name='short_direct_normal',
            glb_SW_up_name='up_short_hemisp',
            glb_LW_dn_name='down_long_hemisp_shaded',
            glb_LW_up_name='up_long_hemisp',
            direct_SW_dn_name='short_direct',
            use_dask=use_dask)

        assert ds_object['qc_down_short_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8]
        assert ds_object['qc_down_short_diffuse_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8]
        assert ds_object['qc_short_direct'].attrs['flag_masks'] == [1, 2, 4, 8]
        assert ds_object['qc_short_direct_normal'].attrs['flag_masks'] == [1, 2, 4, 8]
        assert ds_object['qc_up_short_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8]
        assert ds_object['qc_up_long_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8]

        assert ds_object['qc_up_long_hemisp'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN extremely rare limit of 700.0 W/m^2'

        assert ds_object['qc_down_long_hemisp_shaded'].attrs['flag_meanings'][-1] == \
            'Value greater than BSRN extremely rare limit of 500.0 W/m^2'

        # down_short_hemisp
        result = ds_object.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=1)
        assert np.sum(result) == 100
        result = ds_object.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=2)
        assert np.sum(result) == 26
        result = ds_object.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=3)
        assert np.sum(result) == 337
        result = ds_object.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=4)
        assert np.sum(result) == 66

        # down_short_diffuse_hemisp
        result = ds_object.qcfilter.get_qc_test_mask('down_short_diffuse_hemisp', test_number=1)
        assert np.sum(result) == 50
        result = ds_object.qcfilter.get_qc_test_mask('down_short_diffuse_hemisp', test_number=2)
        assert np.sum(result) == 56
        result = ds_object.qcfilter.get_qc_test_mask('down_short_diffuse_hemisp', test_number=3)
        assert np.sum(result) == 100
        result = ds_object.qcfilter.get_qc_test_mask('down_short_diffuse_hemisp', test_number=4)
        assert np.sum(result) == 90

        # short_direct_normal
        result = ds_object.qcfilter.get_qc_test_mask('short_direct_normal', test_number=1)
        assert np.sum(result) == 46
        result = ds_object.qcfilter.get_qc_test_mask('short_direct_normal', test_number=2)
        assert np.sum(result) == 26
        result = ds_object.qcfilter.get_qc_test_mask('short_direct_normal', test_number=3)
        assert np.sum(result) == 94
        result = ds_object.qcfilter.get_qc_test_mask('short_direct_normal', test_number=4)
        assert np.sum(result) == 38

        # short_direct_normal
        result = ds_object.qcfilter.get_qc_test_mask('short_direct', test_number=1)
        assert np.sum(result) == 41
        result = ds_object.qcfilter.get_qc_test_mask('short_direct', test_number=2)
        assert np.sum(result) == 607
        result = ds_object.qcfilter.get_qc_test_mask('short_direct', test_number=3)
        assert np.sum(result) == 89
        result = ds_object.qcfilter.get_qc_test_mask('short_direct', test_number=4)
        assert np.sum(result) == 79

        # down_long_hemisp_shaded
        result = ds_object.qcfilter.get_qc_test_mask('down_long_hemisp_shaded', test_number=1)
        assert np.sum(result) == 50
        result = ds_object.qcfilter.get_qc_test_mask('down_long_hemisp_shaded', test_number=2)
        assert np.sum(result) == 40
        result = ds_object.qcfilter.get_qc_test_mask('down_long_hemisp_shaded', test_number=3)
        assert np.sum(result) == 89
        result = ds_object.qcfilter.get_qc_test_mask('down_long_hemisp_shaded', test_number=4)
        assert np.sum(result) == 90

        # up_long_hemisp
        result = ds_object.qcfilter.get_qc_test_mask('up_long_hemisp', test_number=1)
        assert np.sum(result) == 50
        result = ds_object.qcfilter.get_qc_test_mask('up_long_hemisp', test_number=2)
        assert np.sum(result) == 40
        result = ds_object.qcfilter.get_qc_test_mask('up_long_hemisp', test_number=3)
        assert np.sum(result) == 89
        result = ds_object.qcfilter.get_qc_test_mask('up_long_hemisp', test_number=4)
        assert np.sum(result) == 90

        # Change data values to trip tests
        ds_object['down_short_diffuse_hemisp'].data[0:100] = \
            ds_object['down_short_diffuse_hemisp'].data[0:100] + 100
        ds_object['up_long_hemisp'].data[0:100] = \
            ds_object['up_long_hemisp'].data[0:100] - 200

        ds_object.qcfilter.bsrn_comparison_tests(
            ['Global over Sum SW Ratio', 'Diffuse Ratio', 'SW up', 'LW down to air temp',
             'LW up to air temp', 'LW down to LW up'],
            gbl_SW_dn_name='down_short_hemisp',
            glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',
            direct_normal_SW_dn_name='short_direct_normal',
            glb_SW_up_name='up_short_hemisp',
            glb_LW_dn_name='down_long_hemisp_shaded',
            glb_LW_up_name='up_long_hemisp',
            air_temp_name='temp_mean',
            test_assessment='Indeterminate',
            lat_name='lat',
            lon_name='lon',
            use_dask=use_dask
        )

        # Ratio of Global over Sum SW
        result = ds_object.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=5)
        assert np.sum(result) == 190

        # Diffuse Ratio
        result = ds_object.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=6)
        assert np.sum(result) == 47

        # Shortwave up comparison
        result = ds_object.qcfilter.get_qc_test_mask('up_short_hemisp', test_number=5)
        assert np.sum(result) == 226

        # Longwave up to air temperature comparison
        result = ds_object.qcfilter.get_qc_test_mask('up_long_hemisp', test_number=5)
        assert np.sum(result) == 290

        # Longwave down to air temperature compaison
        result = ds_object.qcfilter.get_qc_test_mask('down_long_hemisp_shaded', test_number=5)
        assert np.sum(result) == 976

        # Lonwave down to longwave up comparison
        result = ds_object.qcfilter.get_qc_test_mask('down_long_hemisp_shaded', test_number=6)
        assert np.sum(result) == 100
