sync.hh

Go to the documentation of this file.

// -*- c-basic-offset: 4 -*-
#ifndef CLICK_SYNC_HH
#define CLICK_SYNC_HH
#include <click/machine.hh>
#include <click/glue.hh>
#include <click/atomic.hh>
#if CLICK_LINUXMODULE || (CLICK_USERLEVEL && HAVE_MULTITHREAD)
# define CLICK_MULTITHREAD_SPINLOCK 1
#endif
#if CLICK_USERLEVEL && !NDEBUG
# define SPINLOCK_ASSERTLEVEL "<-999>"
#else
# define SPINLOCK_ASSERTLEVEL "<1>"
#endif
CLICK_DECLS
 
 
template <typename T>
class per_thread
{
private:
    struct A_t{
        T v;
    } CLICK_CACHE_ALIGN;
 
    typedef struct A_t AT;
 
    void initialize(unsigned int n, T v) {
        _size = n;
        storage = CLICK_ALIGNED_NEW(AT,_size);
        for (unsigned i = 0; i < n; i++) {
            storage[i].v = v;
        }
    }
 
    //Disable copy constructor. It will always be a user error
    per_thread (const per_thread<T> &);
public:
    explicit per_thread() {
        _size = click_max_cpu_ids();
        storage = CLICK_ALIGNED_NEW(AT,_size);
        for (unsigned i = 0; i < _size; i++) {
            new (&storage[i].v) T;
        }
    }
 
    explicit per_thread(T v) {
        initialize(click_max_cpu_ids(),v);
    }
 
    explicit per_thread(T v, int n) {
        initialize(n,v);
    }
 
    void resize(unsigned int max_cpu_id, T v) {
        CLICK_ALIGNED_DELETE(storage, AT, _size);
        initialize(max_cpu_id,v);
    }
 
    ~per_thread() {
        if (_size) {
            CLICK_ALIGNED_DELETE(storage,AT,_size);
            _size = 0;
        }
    }
 
    inline T* operator->() const {
        return &(storage[click_current_cpu_id()].v);
    }
    inline T& operator*() const {
        return storage[click_current_cpu_id()].v;
    }
 
    inline T& operator+=(const T& add) const {
        storage[click_current_cpu_id()].v += add;
        return storage[click_current_cpu_id()].v;
    }
 
    inline T& operator++() const { // prefix ++
        return ++(storage[click_current_cpu_id()].v);
    }
 
    inline T operator++(int) const { // postfix ++
        return storage[click_current_cpu_id()].v++;
    }
 
    inline T& operator--() const {
        return --(storage[click_current_cpu_id()].v);
    }
 
    inline T operator--(int) const {
        return storage[click_current_cpu_id()].v--;
    }
 
    inline T& operator=(T value) const {
        storage[click_current_cpu_id()].v = value;
        return storage[click_current_cpu_id()].v;
    }
 
    inline void operator=(const per_thread<T>& pt) {
        if (_size != pt._size) {
            if (storage)
               CLICK_ALIGNED_DELETE(storage,AT, _size);
            storage = CLICK_ALIGNED_NEW(AT, _size);
        }
        for (int i = 0; i < pt.weight(); i++) {
            storage[i] = pt.storage[i];
        }
    }
 
    inline void set(T v) {
        storage[click_current_cpu_id()].v = v;
    }
 
    inline void setAll(T v) {
        for (int i = 0; i < click_max_cpu_ids(); i++)
            storage[i].v = v;
    }
 
    inline T& get() const{
        return storage[click_current_cpu_id()].v;
    }
 
    inline const T& cst() const{
        return storage[click_current_cpu_id()].v;
    }
 
    inline T& get_value_for_thread(int thread_id) const{
        return storage[thread_id].v;
    }
 
    inline void set_value_for_thread(int thread_id, T v) {
        storage[thread_id].v = v;
    }
 
    inline T& get_value(int i) const {
        return get_value_for_thread(i);
    }
 
    inline void set_value(int i, T v) {
        set_value_for_thread(i, v);
    }
 
    inline unsigned weight() const {
        return _size;
    }
 
    inline unsigned int get_mapping(int i) {
        return i;
    }
 
    class const_iterator {public:
        const_iterator(const per_thread<T>* t,unsigned pos) {
            _t = t;
            _pos = pos;
        }
        const per_thread<T>* _t;
        unsigned _pos;
        T& operator*() const {
            return _t->get_value(_pos);
        }
        bool operator!=(const const_iterator &o) const {
            return _pos != o._pos;
        }
        void operator++() {
            _pos++;
        }
    };
 
    const_iterator begin() const {
        return const_iterator(this,0);
    }
 
    const_iterator end() const {
        return const_iterator(this,_size);
    }
 
    class iterator {public:
        iterator(per_thread<T>* t,unsigned pos) {
            _t = t;
            _pos = pos;
        }
        const per_thread<T>* _t;
        unsigned _pos;
        T& operator*() const {
            return _t->get_value(_pos);
        }
        bool operator!=(const iterator &o) const {
            return _pos != o._pos;
        }
        void operator++() {
            _pos++;
        }
    };
 
    iterator begin() {
        return iterator(this,0);
    }
 
    iterator end() {
        return iterator(this,_size);
    }
protected:
    AT* storage;
    unsigned _size;
};
 
template <typename T>
class not_per_thread
{
public:
    explicit not_per_thread() {
    }
 
    explicit not_per_thread(T _v) {
        v = _v;
    }
 
    inline T* operator->() {
        return &(v);
    }
 
 
    inline T& operator*() {
        return v;
    }
 
    inline T& get() {
        return v;
    }
 
    inline const T& cst() const {
        return v;
    }
 
    inline constexpr int weight() const {
        return 1;
    }
 
    inline T& get_value(int) {
        return v;
    }
 
    class const_iterator {public:
        const_iterator(const not_per_thread<T>* t,unsigned pos) {
            _t = t;
            _pos = pos;
        }
        const not_per_thread<T>* _t;
        unsigned _pos;
        const T& operator*() const {
            return _t->v;
        }
        bool operator!=(const const_iterator &o) const {
            return _pos != o._pos;
        }
        void operator++() {
            _pos++;
        }
    };
 
    const_iterator begin() const {
        return const_iterator(this,0);
    }
 
    const_iterator end() const {
        return const_iterator(this, 1);
    }
 
    class iterator {public:
        iterator(not_per_thread<T>* t,unsigned pos) {
            _t = t;
            _pos = pos;
        }
        not_per_thread<T>* _t;
        unsigned _pos;
        T& operator*() const {
            return _t->v;
        }
        bool operator!=(const iterator &o) const {
            return _pos != o._pos;
        }
        void operator++() {
            _pos++;
        }
    };
 
    iterator begin() {
        return iterator(this,0);
    }
 
    iterator end() {
        return iterator(this, 1);
    }
private:
    T v;
};
 
 
#define PER_THREAD_SET(pt,value) \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        pt.set_value(i, value); \
    }
 
#define PER_THREAD_POINTER_SET(pt,member,value) \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        pt.get_value(i)->member = value; \
    }
 
#define PER_THREAD_MEMBER_SET(pt,member,value) \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        pt.get_value(i).member = value; \
    }
 
#define PER_THREAD_VECTOR_SET(pt,member,value) \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        for (int j = 0; j < pt.get_value(i).size(); j++) \
            (pt.get_value(i))[j].member = value; \
    }
 
#define PER_THREAD_SUM(type, var, pt) \
    type var = 0; \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        var += pt.get_value(i); \
    }
 
#define PER_THREAD_POINTER_SUM(type, var, pt, member) \
    type var = 0; \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        var += pt.get_value(i)->member; \
    }
 
#define PER_THREAD_MEMBER_SUM(type, var, pt, member) \
    type var = 0; \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        var += pt.get_value(i).member; \
    }
 
#define PER_THREAD_VECTOR_SUM(type, var, pt, index, member) \
    type var = 0; \
    for (unsigned i = 0; i < pt.weight(); i++) { \
        var += pt.get_value(i)[index].member; \
    }
 
 
 
class Spinlock { public:
 
    inline Spinlock();
    inline ~Spinlock();
 
    inline void acquire();
    inline void release();
    inline bool attempt();
    inline bool nested() const;
 
#if CLICK_MULTITHREAD_SPINLOCK
  private:
 
    atomic_uint32_t _lock;
    int32_t _depth;
    click_processor_t _owner;
#endif
 
};
 
inline
Spinlock::Spinlock()
#if CLICK_MULTITHREAD_SPINLOCK
    : _depth(0), _owner(click_invalid_processor())
#endif
{
#if CLICK_MULTITHREAD_SPINLOCK
    _lock = 0;
#endif
}
 
inline
Spinlock::~Spinlock()
{
#if CLICK_MULTITHREAD_SPINLOCK
    if (_depth != 0)
        click_chatter(SPINLOCK_ASSERTLEVEL "Spinlock::~Spinlock(): assertion \"_depth == 0\" failed");
#endif
}
 
inline void
Spinlock::acquire()
{
#if CLICK_MULTITHREAD_SPINLOCK
    click_processor_t my_cpu = click_get_processor();
    if (_owner != my_cpu) {
        while (_lock.swap(1) != 0)
            do {
                click_relax_fence();
            } while (_lock != 0);
        _owner = my_cpu;
    }
    _depth++;
#endif
}
 
inline bool
Spinlock::attempt()
{
#if CLICK_MULTITHREAD_SPINLOCK
    click_processor_t my_cpu = click_get_processor();
    if (_owner != my_cpu) {
        if (_lock.swap(1) != 0) {
            click_put_processor();
            return false;
        }
        _owner = my_cpu;
    }
    _depth++;
    return true;
#else
    return true;
#endif
}
 
inline void
Spinlock::release()
{
#if CLICK_MULTITHREAD_SPINLOCK
    if (unlikely(_owner != click_current_processor()))
        click_chatter(SPINLOCK_ASSERTLEVEL "Spinlock::release(): assertion \"owner == click_current_processor()\" failed");
    if (likely(_depth > 0)) {
        if (--_depth == 0) {
            _owner = click_invalid_processor();
            _lock = 0;
        }
    } else
        click_chatter(SPINLOCK_ASSERTLEVEL "Spinlock::release(): assertion \"_depth > 0\" failed");
    click_put_processor();
#endif
}
 
inline bool
Spinlock::nested() const
{
#if CLICK_MULTITHREAD_SPINLOCK
    return _depth > 1;
#else
    return false;
#endif
}
 
 
class SimpleSpinlock { public:
 
    inline SimpleSpinlock();
    inline ~SimpleSpinlock();
 
    inline void acquire();
    inline void release();
    inline bool attempt();
 
#if CLICK_LINUXMODULE
  private:
    spinlock_t _lock;
#elif CLICK_MULTITHREAD_SPINLOCK
  private:
    atomic_uint32_t _lock;
#endif
 
};
 
inline
SimpleSpinlock::SimpleSpinlock()
{
#if CLICK_LINUXMODULE
    spin_lock_init(&_lock);
#elif CLICK_MULTITHREAD_SPINLOCK
    _lock = 0;
#endif
}
 
inline
SimpleSpinlock::~SimpleSpinlock()
{
}
 
inline void
SimpleSpinlock::acquire()
{
#if CLICK_LINUXMODULE
    spin_lock(&_lock);
#elif CLICK_MULTITHREAD_SPINLOCK
    while (_lock.swap(1) != 0) {
        do {
            click_relax_fence();
        } while (_lock != 0);
    }
#endif
}
 
inline bool
SimpleSpinlock::attempt()
{
#if CLICK_LINUXMODULE
    return spin_trylock(&_lock);
#elif CLICK_MULTITHREAD_SPINLOCK
    return _lock.swap(1) == 0;
#else
    return true;
#endif
}
 
inline void
SimpleSpinlock::release()
{
#if CLICK_LINUXMODULE
    spin_unlock(&_lock);
#elif CLICK_MULTITHREAD_SPINLOCK
    _lock = 0;
#endif
}
 
 
class SpinlockIRQ { public:
 
    inline SpinlockIRQ();
 
#if CLICK_LINUXMODULE
    typedef unsigned long flags_t;
#else
    typedef int flags_t;
#endif
 
    inline flags_t acquire() CLICK_ALWAYS_INLINE;
    inline void release(flags_t) CLICK_ALWAYS_INLINE;
 
#if CLICK_LINUXMODULE
  private:
    spinlock_t _lock;
#elif CLICK_USERLEVEL && HAVE_MULTITHREAD
  private:
    Spinlock _lock;
#endif
 
};
 
inline
SpinlockIRQ::SpinlockIRQ()
{
#if CLICK_LINUXMODULE
    spin_lock_init(&_lock);
#endif
}
 
inline SpinlockIRQ::flags_t
SpinlockIRQ::acquire()
{
#if CLICK_LINUXMODULE
    flags_t flags;
    spin_lock_irqsave(&_lock, flags);
    return flags;
#elif CLICK_USERLEVEL && HAVE_MULTITHREAD
    _lock.acquire();
    return 0;
#else
    return 0;
#endif
}
 
inline void
SpinlockIRQ::release(flags_t flags)
{
#if CLICK_LINUXMODULE
    spin_unlock_irqrestore(&_lock, flags);
#elif CLICK_USERLEVEL && HAVE_MULTITHREAD
    (void) flags;
    _lock.release();
#else
    (void) flags;
#endif
}
 
 
// read-write lock
//
// on read: acquire local read lock
// on write: acquire every read lock
//
// alternatively, we could use a read counter and a write lock. we don't do
// that because we'd like to avoid a cache miss for read acquires. this makes
// reads very fast, and writes more expensive
 
class ReadWriteLock { public:
 
    inline ReadWriteLock();
    inline ~ReadWriteLock();
 
    inline void acquire_read();
    inline bool attempt_read();
    inline void release_read();
    inline void acquire_write();
    inline bool attempt_write();
    inline void release_write();
 
  private:
    // allocate a cache line for every member
    per_thread<Spinlock> _l;
 
};
 
inline
ReadWriteLock::ReadWriteLock()
{
}
 
inline
ReadWriteLock::~ReadWriteLock()
{
}
 
inline void
ReadWriteLock::acquire_read()
{
#if HAVE_MULTITHREAD
    _l->acquire();
#endif
}
 
inline bool
ReadWriteLock::attempt_read()
{
#if HAVE_MULTITHREAD
    bool result = _l->attempt();
    if (!result)
        click_put_processor();
    return result;
#else
    return true;
#endif
}
 
inline void
ReadWriteLock::release_read()
{
#if HAVE_MULTITHREAD
    _l->release();
    click_put_processor();
#endif
}
 
inline void
ReadWriteLock::acquire_write()
{
#if HAVE_MULTITHREAD
    for (unsigned i = 0; i < _l.weight(); i++)
        _l.get_value(i).acquire();
#endif
}
 
inline bool
ReadWriteLock::attempt_write()
{
#if HAVE_MULTITHREAD
    bool all = true;
    unsigned i;
    for (i = 0; i < (unsigned) _l.weight(); i++) {
        if (!(_l.get_value(i).attempt())) {
            all = false;
            break;
        }
    }
    if (!all) {
        for (unsigned j = 0; j < i; j++)
            _l.get_value(j).release();
    }
    return all;
#else
    return true;
#endif
}
 
inline void
ReadWriteLock::release_write()
{
#if HAVE_MULTITHREAD
    for (unsigned i = 0; i < _l.weight(); i++)
        _l.get_value(i).release();
#endif
}
 
class nolock { public:
    nolock() {}
 
    inline void acquire() {};
 
    inline void release() {};
 
    inline bool attempt() { return true; };
 
    inline void read_begin() {}
 
    inline void read_end() {}
 
    inline void read_get() {}
 
    inline void write_begin() {}
 
    inline void write_end() {}
 
    inline void set_max_writers(int) {}
};
 
CLICK_ENDDECLS
#undef SPINLOCK_ASSERTLEVEL
#endif