Click
Loading...
Searching...
No Matches
atomic.hh
Go to the documentation of this file.
1// -*- c-basic-offset: 4 -*-
2#ifndef CLICK_ATOMIC_HH
3#define CLICK_ATOMIC_HH
4#if CLICK_LINUXMODULE
5# include <click/glue.hh>
6#endif
7CLICK_DECLS
8#if CLICK_LINUXMODULE
9# if HAVE_LINUX_ASM_SYSTEM_H
10# include <click/cxxprotect.h>
11CLICK_CXX_PROTECT
12# include <asm/system.h>
13CLICK_CXX_UNPROTECT
14# include <click/cxxunprotect.h>
15# endif
16# define CLICK_ATOMIC_VAL _val.counter
17#else
18# define CLICK_ATOMIC_VAL _val
19#endif
20#if defined(__i386__) || defined(__arch_um__) || defined(__x86_64__)
21# if CLICK_LINUXMODULE || HAVE_MULTITHREAD
22# define CLICK_ATOMIC_X86 1
23# endif
24# if (CLICK_LINUXMODULE && defined(CONFIG_SMP)) || HAVE_MULTITHREAD
25# define CLICK_ATOMIC_LOCK "lock ; "
26# else
27# define CLICK_ATOMIC_LOCK /* nothing */
28# endif
29#endif
30
31
32#if HAVE_MULTITHREAD && HAVE_ATOMIC_BUILTINS
33// #pragma message "atomic.hh: using builtins implementation"
34 # define CLICK_ATOMIC_BUILTINS 1
35 # define CLICK_ATOMIC_MEMORDER __ATOMIC_SEQ_CST
36 // compare_swap methods are not atomic if using builtins. Disable them
37 # define CLICK_BUILTINS_DEPRECATED = delete
38 # define CLICK_BUILTINS_NONDEPRECATED
39 # define CLICK_ATOMIC_COMPARE_SWAP 0
40#else
41 # define CLICK_ATOMIC_BUILTINS 0
42// #pragma message "atomic.hh: Using click own implementation"
43 // compare_swap are truly atomics if correctly implemented (e.g. x86).
44 // In this case, we maintain the original Click's deprecation flags
45 # define CLICK_BUILTINS_DEPRECATED
46 # define CLICK_BUILTINS_NONDEPRECATED CLICK_DEPRECATED
47 # define CLICK_ATOMIC_COMPARE_SWAP 1
48#endif
49
71class atomic_uint32_t { public:
72
73 // No constructors because, unfortunately, GCC generates worse code. Use
74 // operator= instead.
75
76 inline uint32_t value() const;
77 inline uint32_t nonatomic_value() const;
78 inline operator uint32_t() const;
79
80 inline atomic_uint32_t &operator=(uint32_t x);
81
82 inline atomic_uint32_t &operator+=(int32_t delta);
83 inline atomic_uint32_t &operator-=(int32_t delta);
84 inline atomic_uint32_t &operator|=(uint32_t mask);
85 inline atomic_uint32_t &operator&=(uint32_t mask);
86
87 inline void nonatomic_inc();
88 inline void operator++();
89 inline void operator++(int);
90 inline void nonatomic_dec();
91 inline void operator--();
92 inline void operator--(int);
93
94 inline uint32_t swap(uint32_t desired);
95 inline uint32_t fetch_and_add(uint32_t delta);
96 inline bool dec_and_test();
97 inline bool nonatomic_dec_and_test();
98 inline uint32_t compare_swap(uint32_t expected, uint32_t desired) CLICK_BUILTINS_DEPRECATED;
99 inline bool compare_and_swap(uint32_t expected, uint32_t desired) CLICK_BUILTINS_NONDEPRECATED;
100
101 inline static uint32_t swap(volatile uint32_t &x, uint32_t desired);
102 inline static void inc(volatile uint32_t &x);
103 inline static bool dec_and_test(volatile uint32_t &x);
104 inline static uint32_t compare_swap(volatile uint32_t &x, uint32_t expected, uint32_t desired) CLICK_BUILTINS_DEPRECATED;
105 inline static bool compare_and_swap(volatile uint32_t &x, uint32_t expected, uint32_t desired) CLICK_BUILTINS_NONDEPRECATED;
106
107 inline static bool use_builtins(){return CLICK_ATOMIC_BUILTINS;}
108 private:
109
110#if CLICK_LINUXMODULE
111 atomic_t _val;
112#elif HAVE_MULTITHREAD
113 volatile uint32_t _val;
114#else
115 uint32_t _val;
116#endif
117
118};
119
137class atomic_uint64_t { public:
138
139 // No constructors because, unfortunately, GCC generates worse code. Use
140 // operator= instead.
141
142 inline uint64_t value() const;
143 inline uint64_t nonatomic_value() const;
144 inline operator uint64_t() const;
145
146 inline atomic_uint64_t &operator=(uint64_t x);
147
148 inline atomic_uint64_t &operator+=(int64_t delta);
149 inline atomic_uint64_t &operator-=(int64_t delta);
150
151 inline void nonatomic_inc();
152 inline void operator++();
153 inline void operator++(int);
154 inline void nonatomic_dec();
155 inline void operator--();
156 inline void operator--(int);
157
158 inline uint64_t compare_swap(uint64_t expected, uint64_t desired) CLICK_BUILTINS_DEPRECATED;
159 inline uint64_t fetch_and_add(uint64_t delta);
160
161 inline static void add(volatile uint64_t &x, uint64_t delta);
162 inline static uint64_t compare_swap(volatile uint64_t &x, uint64_t expected, uint64_t desired) CLICK_BUILTINS_DEPRECATED;
163
164 inline static bool use_builtins(){return CLICK_ATOMIC_BUILTINS;}
165 private:
166
167#if CLICK_LINUXMODULE
168 atomic64_t _val;
169#elif HAVE_MULTITHREAD
170 volatile uint64_t _val __attribute__((aligned));
171#else
172 uint64_t _val;
173#endif
174
175};
176
178inline uint32_t
179atomic_uint32_t::value() const
180{
181#if CLICK_LINUXMODULE
182 return atomic_read(&_val);
183#else
184 return CLICK_ATOMIC_VAL;
185#endif
186}
187
189inline uint32_t
190atomic_uint32_t::nonatomic_value() const
191{
192 return *((uint32_t*)&_val);
193}
194
196inline
197atomic_uint32_t::operator uint32_t() const
198{
199 return value();
200}
201
203inline atomic_uint32_t &
204atomic_uint32_t::operator=(uint32_t x)
205{
206#if CLICK_LINUXMODULE
207 atomic_set(&_val, x);
208#else
209 CLICK_ATOMIC_VAL = x;
210#endif
211 return *this;
212}
213
215inline atomic_uint32_t &
216atomic_uint32_t::operator+=(int32_t delta)
217{
218#if CLICK_LINUXMODULE
219 atomic_add(delta, &_val);
220#elif CLICK_ATOMIC_BUILTINS
221 __atomic_add_fetch(&CLICK_ATOMIC_VAL, delta, CLICK_ATOMIC_MEMORDER);
222#elif CLICK_ATOMIC_X86
223 asm volatile (CLICK_ATOMIC_LOCK "addl %1,%0"
224 : "=m" (CLICK_ATOMIC_VAL)
225 : "r" (delta), "m" (CLICK_ATOMIC_VAL)
226 : "cc");
227#else
228 CLICK_ATOMIC_VAL += delta;
229#endif
230 return *this;
231}
232
234inline atomic_uint32_t &
235atomic_uint32_t::operator-=(int32_t delta)
236{
237#if CLICK_LINUXMODULE
238 atomic_sub(delta, &_val);
239#elif CLICK_ATOMIC_BUILTINS
240 __atomic_sub_fetch(&CLICK_ATOMIC_VAL, delta, CLICK_ATOMIC_MEMORDER);
241#elif CLICK_ATOMIC_X86
242 asm volatile (CLICK_ATOMIC_LOCK "subl %1,%0"
243 : "=m" (CLICK_ATOMIC_VAL)
244 : "r" (delta), "m" (CLICK_ATOMIC_VAL)
245 : "cc");
246#else
247 CLICK_ATOMIC_VAL -= delta;
248#endif
249 return *this;
250}
251
253inline atomic_uint32_t &
254atomic_uint32_t::operator|=(uint32_t mask)
255{
256#if CLICK_LINUXMODULE && HAVE_LINUX_ATOMIC_SET_MASK
257 atomic_set_mask(mask, &_val);
258#elif CLICK_ATOMIC_BUILTINS
259 __atomic_or_fetch(&CLICK_ATOMIC_VAL, mask, CLICK_ATOMIC_MEMORDER);
260#elif CLICK_ATOMIC_X86
261 asm volatile (CLICK_ATOMIC_LOCK "orl %1,%0"
262 : "=m" (CLICK_ATOMIC_VAL)
263 : "r" (mask), "m" (CLICK_ATOMIC_VAL)
264 : "cc");
265#elif CLICK_LINUXMODULE
266# warning "using nonatomic approximation for atomic_uint32_t::operator|="
267 unsigned long flags;
268 local_irq_save(flags);
269 CLICK_ATOMIC_VAL |= mask;
270 local_irq_restore(flags);
271#else
272 CLICK_ATOMIC_VAL |= mask;
273#endif
274 return *this;
275}
276
278inline atomic_uint32_t &
279atomic_uint32_t::operator&=(uint32_t mask)
280{
281#if CLICK_LINUXMODULE && HAVE_LINUX_ATOMIC_SET_MASK
282 atomic_clear_mask(~mask, &_val);
283#elif CLICK_ATOMIC_BUILTINS
284 __atomic_and_fetch(&CLICK_ATOMIC_VAL, mask, CLICK_ATOMIC_MEMORDER);
285#elif CLICK_ATOMIC_X86
286 asm volatile (CLICK_ATOMIC_LOCK "andl %1,%0"
287 : "=m" (CLICK_ATOMIC_VAL)
288 : "r" (mask), "m" (CLICK_ATOMIC_VAL)
289 : "cc");
290#elif CLICK_LINUXMODULE
291# warning "using nonatomic approximation for atomic_uint32_t::operator&="
292 unsigned long flags;
293 local_irq_save(flags);
294 CLICK_ATOMIC_VAL &= mask;
295 local_irq_restore(flags);
296#else
297 CLICK_ATOMIC_VAL &= mask;
298#endif
299 return *this;
300}
301
303inline void
304atomic_uint32_t::inc(volatile uint32_t &x)
305{
306#if CLICK_LINUXMODULE
307 static_assert(sizeof(atomic_t) == sizeof(x), "atomic_t expected to take 32 bits.");
308 atomic_inc((atomic_t *) &x);
309#elif CLICK_ATOMIC_BUILTINS
310 __atomic_add_fetch(&x, 1, CLICK_ATOMIC_MEMORDER);
311#elif CLICK_ATOMIC_X86
312 asm volatile (CLICK_ATOMIC_LOCK "incl %0"
313 : "=m" (x)
314 : "m" (x)
315 : "cc");
316#else
317 x++;
318#endif
319}
320
322inline void
323atomic_uint32_t::nonatomic_inc()
324{
325 CLICK_ATOMIC_VAL++;
326}
327
329inline void
330atomic_uint32_t::operator++()
331{
332#if CLICK_LINUXMODULE
333 atomic_inc(&_val);
334#elif CLICK_ATOMIC_BUILTINS
335 __atomic_add_fetch(&_val, 1, CLICK_ATOMIC_MEMORDER);
336#elif CLICK_ATOMIC_X86
337 asm volatile (CLICK_ATOMIC_LOCK "incl %0"
338 : "=m" (CLICK_ATOMIC_VAL)
339 : "m" (CLICK_ATOMIC_VAL)
340 : "cc");
341#else
342 CLICK_ATOMIC_VAL++;
343#endif
344}
345
347inline void
348atomic_uint32_t::operator++(int)
349{
350#if CLICK_LINUXMODULE
351 atomic_inc(&_val);
352#elif CLICK_ATOMIC_BUILTINS
353 __atomic_add_fetch(&CLICK_ATOMIC_VAL, 1, CLICK_ATOMIC_MEMORDER);
354#elif CLICK_ATOMIC_X86
355 asm volatile (CLICK_ATOMIC_LOCK "incl %0"
356 : "=m" (CLICK_ATOMIC_VAL)
357 : "m" (CLICK_ATOMIC_VAL)
358 : "cc");
359#else
360 CLICK_ATOMIC_VAL++;
361#endif
362}
363
365inline void
366atomic_uint32_t::operator--()
367{
368#if CLICK_LINUXMODULE
369 atomic_dec(&_val);
370#elif CLICK_ATOMIC_BUILTINS
371 __atomic_sub_fetch(&CLICK_ATOMIC_VAL, 1, CLICK_ATOMIC_MEMORDER);
372#elif CLICK_ATOMIC_X86
373 asm volatile (CLICK_ATOMIC_LOCK "decl %0"
374 : "=m" (CLICK_ATOMIC_VAL)
375 : "m" (CLICK_ATOMIC_VAL)
376 : "cc");
377#else
378 CLICK_ATOMIC_VAL--;
379#endif
380}
381
383inline void
384atomic_uint32_t::nonatomic_dec()
385{
386 CLICK_ATOMIC_VAL--;
387}
388
390inline void
391atomic_uint32_t::operator--(int)
392{
393#if CLICK_LINUXMODULE
394 atomic_dec(&_val);
395#elif CLICK_ATOMIC_BUILTINS
396 __atomic_sub_fetch(&CLICK_ATOMIC_VAL, 1, CLICK_ATOMIC_MEMORDER);
397#elif CLICK_ATOMIC_X86
398 asm volatile (CLICK_ATOMIC_LOCK "decl %0"
399 : "=m" (CLICK_ATOMIC_VAL)
400 : "m" (CLICK_ATOMIC_VAL)
401 : "cc");
402#else
403 CLICK_ATOMIC_VAL--;
404#endif
405}
406
417inline uint32_t
418atomic_uint32_t::swap(volatile uint32_t &x, uint32_t desired)
419{
420#if CLICK_ATOMIC_BUILTINS
421 uint32_t actual = x;
422 __atomic_exchange(&x, &desired, &actual, CLICK_ATOMIC_MEMORDER);
423 return actual;
424
425#elif CLICK_ATOMIC_X86
426 asm volatile ("xchgl %0,%1"
427 : "=r" (desired), "=m" (x)
428 : "0" (desired), "m" (x)
429 : "memory");
430 return desired;
431#elif CLICK_LINUXMODULE && defined(xchg)
432 return xchg(&x, desired);
433#elif CLICK_LINUXMODULE
434# error "need xchg for atomic_uint32_t::swap"
435#else
436 uint32_t actual = x;
437 x = desired;
438 return actual;
439#endif
440}
441
452inline uint32_t
453atomic_uint32_t::swap(uint32_t desired)
454{
455#if CLICK_LINUXMODULE && defined(xchg)
456 return atomic_xchg(&_val, desired);
457#elif CLICK_LINUXMODULE
458# error "need xchg for atomic_uint32_t::swap"
459#else
460 return swap(CLICK_ATOMIC_VAL, desired);
461#endif
462}
463
472inline uint32_t
473atomic_uint32_t::fetch_and_add(uint32_t delta)
474{
475#if CLICK_ATOMIC_BUILTINS
476 return __atomic_fetch_add(&_val, delta, CLICK_ATOMIC_MEMORDER);
477#elif CLICK_ATOMIC_X86
478 asm volatile (CLICK_ATOMIC_LOCK "xaddl %0,%1"
479 : "=r" (delta), "=m" (CLICK_ATOMIC_VAL)
480 : "0" (delta), "m" (CLICK_ATOMIC_VAL)
481 : "cc");
482 return delta;
483#elif CLICK_LINUXMODULE && HAVE_LINUX_ATOMIC_ADD_RETURN
484 return atomic_add_return(&_val, delta) - delta;
485#elif CLICK_LINUXMODULE
486# warning "using nonatomic approximation for atomic_uint32_t::fetch_and_add"
487 unsigned long flags;
488 local_irq_save(flags);
489 uint32_t old_value = value();
490 CLICK_ATOMIC_VAL += delta;
491 local_irq_restore(flags);
492 return old_value;
493#else
494 uint32_t old_value = value();
495 CLICK_ATOMIC_VAL += delta;
496 return old_value;
497#endif
498}
499
508inline bool
509atomic_uint32_t::dec_and_test(volatile uint32_t &x)
510{
511#if CLICK_LINUXMODULE
512 static_assert(sizeof(atomic_t) == sizeof(x), "atomic_t expected to take 32 bits.");
513 return atomic_dec_and_test((atomic_t *) &x);
514#elif CLICK_ATOMIC_BUILTINS
515 return __atomic_sub_fetch(&x,1 , CLICK_ATOMIC_MEMORDER) == 0;
516#elif CLICK_ATOMIC_X86
517 uint8_t result;
518 asm volatile (CLICK_ATOMIC_LOCK "decl %0 ; sete %1"
519 : "=m" (x), "=qm" (result)
520 : "m" (x)
521 : "cc");
522 return result;
523#else
524 return (--x == 0);
525#endif
526}
527
544#if CLICK_ATOMIC_COMPARE_SWAP
545inline uint32_t
546atomic_uint32_t::compare_swap(volatile uint32_t &x, uint32_t expected, uint32_t desired)
547{
548#if CLICK_ATOMIC_BUILTINS
549# warning "compare_swap is not truly atomic with system builtins"
550 return __atomic_compare_exchange(&x,&expected,&desired,0,CLICK_ATOMIC_MEMORDER, CLICK_ATOMIC_MEMORDER)
551 ? expected : x;
552#elif CLICK_ATOMIC_X86
553 asm volatile (CLICK_ATOMIC_LOCK "cmpxchgl %2,%1"
554 : "=a" (expected), "=m" (x)
555 : "r" (desired), "0" (expected), "m" (x)
556 : "cc", "memory");
557 return expected;
558#elif CLICK_LINUXMODULE && defined(cmpxchg)
559 return cmpxchg(&x, expected, desired);
560#elif CLICK_LINUXMODULE
561# warning "using nonatomic approximation for atomic_uint32_t::compare_and_swap"
562 unsigned long flags;
563 local_irq_save(flags);
564 uint32_t actual = x;
565 if (actual == expected)
566 x = desired;
567 local_irq_restore(flags);
568 return actual;
569#else
570 uint32_t actual = x;
571 if (actual == expected)
572 x = desired;
573 return actual;
574#endif
575}
576#endif
577
595inline bool
596atomic_uint32_t::compare_and_swap(volatile uint32_t &x, uint32_t expected, uint32_t desired)
597{
598#if CLICK_ATOMIC_BUILTINS
599 return __atomic_compare_exchange(&x,&expected,&desired,0,CLICK_ATOMIC_MEMORDER, CLICK_ATOMIC_MEMORDER);
600
601#elif CLICK_ATOMIC_X86
602 asm volatile (CLICK_ATOMIC_LOCK "cmpxchgl %2,%0 ; sete %%al"
603 : "=m" (x), "=a" (expected)
604 : "r" (desired), "m" (x), "a" (expected)
605 : "cc", "memory");
606 return (uint8_t) expected;
607#elif CLICK_LINUXMODULE && defined(cmpxchg)
608 return cmpxchg(&x, expected, desired) == expected;
609#elif CLICK_LINUXMODULE
610# warning "using nonatomic approximation for atomic_uint32_t::compare_and_swap"
611 unsigned long flags;
612 local_irq_save(flags);
613 uint32_t old_value = x;
614 if (old_value == expected)
615 x = desired;
616 local_irq_restore(flags);
617 return old_value == expected;
618#else
619 uint32_t old_value = x;
620 if (old_value == expected)
621 x = desired;
622 return old_value == expected;
623#endif
624}
625
634inline bool
635atomic_uint32_t::dec_and_test()
636{
637#if CLICK_LINUXMODULE
638 return atomic_dec_and_test(&_val);
639#elif CLICK_ATOMIC_BUILTINS
640 return __atomic_sub_fetch(&_val, 1, CLICK_ATOMIC_MEMORDER) == 0;
641#elif CLICK_ATOMIC_X86
642 uint8_t result;
643 asm volatile (CLICK_ATOMIC_LOCK "decl %0 ; sete %1"
644 : "=m" (CLICK_ATOMIC_VAL), "=qm" (result)
645 : "m" (CLICK_ATOMIC_VAL)
646 : "cc");
647 return result;
648#else
649 return (--CLICK_ATOMIC_VAL == 0);
650#endif
651}
652
661inline bool
662atomic_uint32_t::nonatomic_dec_and_test()
663{
664 CLICK_ATOMIC_VAL--;
665 return CLICK_ATOMIC_VAL == 0;
666}
667
683#if CLICK_ATOMIC_COMPARE_SWAP
684inline uint32_t
685atomic_uint32_t::compare_swap(uint32_t expected, uint32_t desired)
686{
687#if CLICK_ATOMIC_BUILTINS
688# warning "compare_swap is not truly atomic with system builtins"
689 return __atomic_compare_exchange(&_val, &expected, &desired, 0, CLICK_ATOMIC_MEMORDER, CLICK_ATOMIC_MEMORDER)
690 ? expected: _val;
691#elif CLICK_ATOMIC_X86
692 asm volatile (CLICK_ATOMIC_LOCK "cmpxchgl %2,%1"
693 : "=a" (expected), "=m" (CLICK_ATOMIC_VAL)
694 : "r" (desired), "0" (expected), "m" (CLICK_ATOMIC_VAL)
695 : "cc", "memory");
696 return expected;
697#elif CLICK_LINUXMODULE && HAVE_LINUX_ATOMIC_CMPXCHG
698 return atomic_cmpxchg(&_val, expected, desired);
699#elif CLICK_LINUXMODULE
700# warning "using nonatomic approximation for atomic_uint32_t::compare_swap"
701 unsigned long flags;
702 local_irq_save(flags);
703 uint32_t actual = value();
704 if (actual == expected)
705 CLICK_ATOMIC_VAL = desired;
706 local_irq_restore(flags);
707 return actual;
708#else
709 uint32_t actual = value();
710 if (actual == expected)
711 CLICK_ATOMIC_VAL = desired;
712 return actual;
713#endif
714}
715#endif
716
733inline bool
734atomic_uint32_t::compare_and_swap(uint32_t expected, uint32_t desired)
735{
736#if CLICK_ATOMIC_BUILTINS
737 return __atomic_compare_exchange(&_val, &expected, &desired, 0, CLICK_ATOMIC_MEMORDER, CLICK_ATOMIC_MEMORDER);
738#elif CLICK_ATOMIC_X86
739 asm volatile (CLICK_ATOMIC_LOCK "cmpxchgl %2,%0 ; sete %%al"
740 : "=m" (CLICK_ATOMIC_VAL), "=a" (expected)
741 : "r" (desired), "m" (CLICK_ATOMIC_VAL), "a" (expected)
742 : "cc", "memory");
743 return (uint8_t) expected;
744#elif CLICK_LINUXMODULE && HAVE_LINUX_ATOMIC_CMPXCHG
745 return atomic_cmpxchg(&_val, expected, desired) == expected;
746#elif CLICK_LINUXMODULE
747# warning "using nonatomic approximation for atomic_uint32_t::compare_and_swap"
748 unsigned long flags;
749 local_irq_save(flags);
750 uint32_t old_value = value();
751 if (old_value == expected)
752 CLICK_ATOMIC_VAL = desired;
753 local_irq_restore(flags);
754 return old_value == expected;
755#else
756 uint32_t old_value = value();
757 if (old_value == expected)
758 CLICK_ATOMIC_VAL = desired;
759 return old_value == expected;
760#endif
761}
762
763inline uint32_t
764operator+(const atomic_uint32_t &a, const atomic_uint32_t &b)
765{
766 return a.value() + b.value();
767}
768
769inline uint32_t
770operator-(const atomic_uint32_t &a, const atomic_uint32_t &b)
771{
772 return a.value() - b.value();
773}
774
775inline bool
776operator==(const atomic_uint32_t &a, const atomic_uint32_t &b)
777{
778 return a.value() == b.value();
779}
780
781inline bool
782operator!=(const atomic_uint32_t &a, const atomic_uint32_t &b)
783{
784 return a.value() != b.value();
785}
786
787inline bool
788operator>(const atomic_uint32_t &a, const atomic_uint32_t &b)
789{
790 return a.value() > b.value();
791}
792
793inline bool
794operator<(const atomic_uint32_t &a, const atomic_uint32_t &b)
795{
796 return a.value() < b.value();
797}
798
799inline bool
800operator>=(const atomic_uint32_t &a, const atomic_uint32_t &b)
801{
802 return a.value() >= b.value();
803}
804
805inline bool
806operator<=(const atomic_uint32_t &a, const atomic_uint32_t &b)
807{
808 return a.value() <= b.value();
809}
810
811typedef atomic_uint32_t uatomic32_t;
812
813//Atomic 64
814
816inline uint64_t
817atomic_uint64_t::value() const
818{
819#if CLICK_LINUXMODULE
820 return atomic64_read(&_val);
821#else
822 return CLICK_ATOMIC_VAL;
823#endif
824}
825
827inline uint64_t
828atomic_uint64_t::nonatomic_value() const
829{
830 return *((uint64_t*)&_val);
831}
832
834inline
835atomic_uint64_t::operator uint64_t() const
836{
837 return value();
838}
839
841inline atomic_uint64_t &
842atomic_uint64_t::operator=(uint64_t x)
843{
844#if CLICK_LINUXMODULE
845 atomic64_set(&_val, x);
846#else
847 CLICK_ATOMIC_VAL = x;
848#endif
849 return *this;
850}
851
853inline atomic_uint64_t &
854atomic_uint64_t::operator+=(int64_t delta)
855{
856#if CLICK_LINUXMODULE
857 atomic64_add(delta, &_val);
858#elif CLICK_ATOMIC_BUILTINS
859 __atomic_add_fetch(&_val, delta, CLICK_ATOMIC_MEMORDER);
860#elif CLICK_ATOMIC_X86
861 asm volatile (CLICK_ATOMIC_LOCK "addq %1,%0"
862 : "=m" (CLICK_ATOMIC_VAL)
863 : "r" (delta), "m" (CLICK_ATOMIC_VAL)
864 : "cc");
865#else
866 CLICK_ATOMIC_VAL += delta;
867#endif
868 return *this;
869}
870
872inline void
873atomic_uint64_t::add(volatile uint64_t &x, uint64_t delta)
874{
875#if CLICK_LINUXMODULE
876 atomic64_add(delta, (atomic64_t*)&x);
877#elif CLICK_ATOMIC_BUILTINS
878 __atomic_add_fetch(&x, delta, CLICK_ATOMIC_MEMORDER);
879#elif CLICK_ATOMIC_X86
880 asm volatile (CLICK_ATOMIC_LOCK "addq %1,%0"
881 : "=m" (x)
882 : "r" (delta), "m" (x)
883 : "cc");
884#else
885 x += delta;
886#endif
887}
888
890inline atomic_uint64_t &
891atomic_uint64_t::operator-=(int64_t delta)
892{
893#if CLICK_LINUXMODULE
894 atomic64_sub(delta, &_val);
895#elif CLICK_ATOMIC_BUILTINS
896 __atomic_sub_fetch(&CLICK_ATOMIC_VAL, delta, CLICK_ATOMIC_MEMORDER);
897#elif CLICK_ATOMIC_X86
898 asm volatile (CLICK_ATOMIC_LOCK "subq %1,%0"
899 : "=m" (CLICK_ATOMIC_VAL)
900 : "r" (delta), "m" (CLICK_ATOMIC_VAL)
901 : "cc");
902#else
903 CLICK_ATOMIC_VAL -= delta;
904#endif
905 return *this;
906}
907
908
910inline void
911atomic_uint64_t::nonatomic_inc()
912{
913 CLICK_ATOMIC_VAL++;
914}
915
917inline void
918atomic_uint64_t::operator++()
919{
920#if CLICK_LINUXMODULE
921 atomic64_inc(&_val);
922#elif CLICK_ATOMIC_X86
923 asm volatile (CLICK_ATOMIC_LOCK "incq %0"
924 : "=m" (CLICK_ATOMIC_VAL)
925 : "m" (CLICK_ATOMIC_VAL)
926 : "cc");
927#else
928 CLICK_ATOMIC_VAL++;
929#endif
930}
931
933inline void
934atomic_uint64_t::operator++(int)
935{
936#if CLICK_LINUXMODULE
937 atomic64_inc(&_val);
938#elif CLICK_ATOMIC_BUILTINS
939 __atomic_add_fetch(&CLICK_ATOMIC_VAL, 1, CLICK_ATOMIC_MEMORDER);
940#elif CLICK_ATOMIC_X86
941 asm volatile (CLICK_ATOMIC_LOCK "incq %0"
942 : "=m" (CLICK_ATOMIC_VAL)
943 : "m" (CLICK_ATOMIC_VAL)
944 : "cc");
945#else
946 CLICK_ATOMIC_VAL++;
947#endif
948}
949
951inline void
952atomic_uint64_t::operator--()
953{
954#if CLICK_LINUXMODULE
955 atomic64_dec(&_val);
956#elif CLICK_ATOMIC_BUILTINS
957 __atomic_sub_fetch(&CLICK_ATOMIC_VAL, 1, CLICK_ATOMIC_MEMORDER);
958#elif CLICK_ATOMIC_X86
959 asm volatile (CLICK_ATOMIC_LOCK "decq %0"
960 : "=m" (CLICK_ATOMIC_VAL)
961 : "m" (CLICK_ATOMIC_VAL)
962 : "cc");
963#else
964 CLICK_ATOMIC_VAL--;
965#endif
966}
967
969inline void
970atomic_uint64_t::nonatomic_dec()
971{
972 CLICK_ATOMIC_VAL--;
973}
974
976inline void
977atomic_uint64_t::operator--(int)
978{
979#if CLICK_LINUXMODULE
980 atomic64_dec(&_val);
981#elif CLICK_ATOMIC_BUILTINS
982 __atomic_sub_fetch(&CLICK_ATOMIC_VAL, 1, CLICK_ATOMIC_MEMORDER);
983#elif CLICK_ATOMIC_X86
984 asm volatile (CLICK_ATOMIC_LOCK "decq %0"
985 : "=m" (CLICK_ATOMIC_VAL)
986 : "m" (CLICK_ATOMIC_VAL)
987 : "cc");
988#else
989 CLICK_ATOMIC_VAL--;
990#endif
991}
992
1009#if CLICK_ATOMIC_COMPARE_SWAP
1010inline uint64_t
1011atomic_uint64_t::compare_swap(volatile uint64_t &x, uint64_t expected, uint64_t desired)
1012{
1013#if CLICK_ATOMIC_BUILTINS
1014# warning "compare_swap is not truly atomic with system builtins"
1015 return __atomic_compare_exchange(&x,&expected,&desired,0,CLICK_ATOMIC_MEMORDER, CLICK_ATOMIC_MEMORDER)
1016 ? expected : x;
1017#elif CLICK_ATOMIC_X86
1018 asm volatile (CLICK_ATOMIC_LOCK "cmpxchgq %2,%1"
1019 : "=a" (expected), "=m" (x)
1020 : "r" (desired), "0" (expected), "m" (x)
1021 : "cc", "memory");
1022 return expected;
1023#elif CLICK_LINUXMODULE && defined(cmpxchg)
1024 return cmpxchg(&x, expected, desired);
1025#elif CLICK_LINUXMODULE
1026# warning "using nonatomic approximation for atomic_uint§2_t::compare_and_swap"
1027 unsigned long flags;
1028 local_irq_save(flags);
1029 uint64_t actual = x;
1030 if (actual == expected)
1031 x = desired;
1032 local_irq_restore(flags);
1033 return actual;
1034#else
1035 uint64_t actual = x;
1036 if (actual == expected)
1037 x = desired;
1038 return actual;
1039#endif
1040}
1041
1042inline uint64_t
1043atomic_uint64_t::compare_swap(uint64_t expected, uint64_t desired) {
1044#if CLICK_LINUXMODULE && defined(cmpxchg)
1045 return atomic64_cmpxchg(&_val, expected, desired);
1046#else
1047 return atomic_uint64_t::compare_swap(*(volatile uint64_t*)&CLICK_ATOMIC_VAL, expected, desired);
1048#endif
1049}
1050#endif
1051
1060inline uint64_t
1061atomic_uint64_t::fetch_and_add(uint64_t delta)
1062{
1063#if CLICK_ATOMIC_BUILTINS
1064 return __atomic_fetch_add(&_val, delta, CLICK_ATOMIC_MEMORDER);
1065#elif CLICK_ATOMIC_X86
1066 asm volatile (CLICK_ATOMIC_LOCK "xaddq %0,%1"
1067 : "=r" (delta), "=m" (CLICK_ATOMIC_VAL)
1068 : "0" (delta), "m" (CLICK_ATOMIC_VAL)
1069 : "cc");
1070 return delta;
1071#elif CLICK_LINUXMODULE && HAVE_LINUX_ATOMIC_ADD_RETURN
1072 return atomic64_add_return(&_val, delta) - delta;
1073#elif CLICK_LINUXMODULE
1074# warning "using nonatomic approximation for atomic_uint64_t::fetch_and_add"
1075 unsigned long flags;
1076 local_irq_save(flags);
1077 uint64_t old_value = value();
1078 CLICK_ATOMIC_VAL += delta;
1079 local_irq_restore(flags);
1080 return old_value;
1081#else
1082 uint64_t old_value = value();
1083 CLICK_ATOMIC_VAL += delta;
1084 return old_value;
1085#endif
1086}
1087
1088
1089
1090class nonatomic_uint32_t { public:
1091
1092 // Fake atomic to drop-in for atomic uint32_t. Not actually atomic.
1093
1094 inline uint32_t value() const { return _val; }
1095 inline uint32_t nonatomic_value() const { return _val; }
1096 inline operator uint32_t() const { return _val; }
1097
1098 inline nonatomic_uint32_t &operator=(uint32_t x) { _val = x; return *this; }
1099
1100 inline nonatomic_uint32_t &operator+=(int32_t delta) { _val += delta; return *this; }
1101 inline nonatomic_uint32_t &operator-=(int32_t delta) { _val -= delta; return *this; }
1102 inline nonatomic_uint32_t &operator|=(uint32_t mask) { _val |= mask; return *this; }
1103 inline nonatomic_uint32_t &operator&=(uint32_t mask) { _val &= mask; return *this; }
1104
1105 inline void nonatomic_inc() { ++_val; }
1106 inline void operator++() { ++_val; }
1107 inline void operator++(int) { ++_val; }
1108 inline void nonatomic_dec() { --_val; }
1109 inline void operator--() { --_val; }
1110 inline void operator--(int) { --_val; }
1111
1112 inline uint32_t swap(uint32_t desired) { uint32_t old = _val; _val = desired; return old; }
1113 inline uint32_t fetch_and_add(uint32_t delta) { uint32_t old = _val; _val += delta; return old; }
1114 inline bool dec_and_test() { return --_val == 0; }
1115 inline bool nonatomic_dec_and_test() { return --_val == 0; }
1116 inline uint32_t compare_swap(uint32_t expected, uint32_t desired) { uint32_t old = _val; if (_val == expected) _val = desired; return old; }
1117 inline bool compare_and_swap(uint32_t expected, uint32_t desired) { if (_val == expected) { _val = desired; return true; } return false; }
1118
1119 inline static uint32_t swap(volatile uint32_t &x, uint32_t desired) { uint32_t old = x; x = desired; return old; }
1120 inline static void inc(volatile uint32_t &x) { ++x; }
1121 inline static bool dec_and_test(volatile uint32_t &x) { return --x == 0; }
1122 inline static uint32_t compare_swap(volatile uint32_t &x, uint32_t expected, uint32_t desired) { uint32_t old = x; if (x == expected) x = desired; return old; }
1123 inline static bool compare_and_swap(volatile uint32_t &x, uint32_t expected, uint32_t desired) { if (x == expected) { x = desired; return true; } return false; }
1124
1125 inline static bool use_builtins() { return false; }
1126 private:
1127
1128 uint32_t _val;
1129};
1130
1131CLICK_ENDDECLS
1132#endif
atomic_uint32_t
A 32-bit integer with support for atomic operations.
Definition atomic.hh:71
atomic_uint32_t::dec_and_test
bool dec_and_test()
Atomically decrement the value, returning true if the new value is 0.
Definition atomic.hh:635
atomic_uint32_t::operator&=
atomic_uint32_t & operator&=(uint32_t mask)
Atomically bitwise-and the value with mask.
Definition atomic.hh:279
atomic_uint32_t::nonatomic_dec
void nonatomic_dec()
Decrement the value.
Definition atomic.hh:384
atomic_uint32_t::operator++
void operator++()
Atomically increment the value.
Definition atomic.hh:330
atomic_uint32_t::swap
uint32_t swap(uint32_t desired)
Atomically assign the value to desired, returning the old value.
Definition atomic.hh:453
atomic_uint32_t::operator--
void operator--()
Atomically decrement the value.
Definition atomic.hh:366
atomic_uint32_t::operator-=
atomic_uint32_t & operator-=(int32_t delta)
Atomically subtract delta from the value.
Definition atomic.hh:235
atomic_uint32_t::nonatomic_inc
void nonatomic_inc()
Increment the value.
Definition atomic.hh:323
atomic_uint32_t::fetch_and_add
uint32_t fetch_and_add(uint32_t delta)
Atomically add delta to the value, returning the old value.
Definition atomic.hh:473
atomic_uint32_t::compare_and_swap
bool compare_and_swap(uint32_t expected, uint32_t desired) CLICK_BUILTINS_NONDEPRECATED
Perform a compare-and-swap operation.
Definition atomic.hh:734
atomic_uint32_t::value
uint32_t value() const
Return the value.
Definition atomic.hh:179
atomic_uint32_t::inc
static void inc(volatile uint32_t &x)
Atomically increment value x.
Definition atomic.hh:304
atomic_uint32_t::nonatomic_dec_and_test
bool nonatomic_dec_and_test()
Decrement the value, returning true if the new value is 0.
Definition atomic.hh:662
atomic_uint32_t::nonatomic_value
uint32_t nonatomic_value() const
Return the value.
Definition atomic.hh:190
atomic_uint32_t::operator|=
atomic_uint32_t & operator|=(uint32_t mask)
Atomically bitwise-or the value with mask.
Definition atomic.hh:254
atomic_uint32_t::operator=
atomic_uint32_t & operator=(uint32_t x)
Set the value to x.
Definition atomic.hh:204
atomic_uint32_t::operator+=
atomic_uint32_t & operator+=(int32_t delta)
Atomically add delta to the value.
Definition atomic.hh:216
atomic_uint64_t
A 64-bit integer with support for atomic operations.
Definition atomic.hh:137
atomic_uint64_t::operator+=
atomic_uint64_t & operator+=(int64_t delta)
Atomically add delta to the value.
Definition atomic.hh:854
atomic_uint64_t::operator-=
atomic_uint64_t & operator-=(int64_t delta)
Atomically subtract delta from the value.
Definition atomic.hh:891
atomic_uint64_t::operator++
void operator++()
Atomically increment the value.
Definition atomic.hh:918
atomic_uint64_t::add
static void add(volatile uint64_t &x, uint64_t delta)
Atomically increment value x.
Definition atomic.hh:873
atomic_uint64_t::operator=
atomic_uint64_t & operator=(uint64_t x)
Set the value to x.
Definition atomic.hh:842
atomic_uint64_t::nonatomic_value
uint64_t nonatomic_value() const
Return the value.
Definition atomic.hh:828
atomic_uint64_t::nonatomic_inc
void nonatomic_inc()
Increment the value.
Definition atomic.hh:911
atomic_uint64_t::fetch_and_add
uint64_t fetch_and_add(uint64_t delta)
Perform a compare-and-swap operation.
Definition atomic.hh:1061
atomic_uint64_t::value
uint64_t value() const
Return the value.
Definition atomic.hh:817
atomic_uint64_t::nonatomic_dec
void nonatomic_dec()
Decrement the value.
Definition atomic.hh:970
atomic_uint64_t::operator--
void operator--()
Atomically decrement the value.
Definition atomic.hh:952
Generated on Mon Nov 2 2026 11:59:47 for Click by doxygen 1.9.8