diff options
author | Stephen Hemminger <stephen.hemminger@vyatta.com> | 2010-11-12 10:41:31 -0800 |
---|---|---|
committer | Stephen Hemminger <stephen.hemminger@vyatta.com> | 2010-11-12 11:07:09 -0800 |
commit | ac9042acae2579b03d39fa6d5b7a8dbef1a8a048 (patch) | |
tree | d71efb8490456c3b1ec74c6e9cac3e517633e024 | |
parent | 500c844e27d7ff989ae73a8cff506552844cdf70 (diff) | |
download | vyatta-cfg-system-ac9042acae2579b03d39fa6d5b7a8dbef1a8a048.tar.gz vyatta-cfg-system-ac9042acae2579b03d39fa6d5b7a8dbef1a8a048.zip |
New IRQ affinity script and RPS support
Replace old script with new cleaner script that handles both
IRQ affinity and Receive Packet Steering. Instead of two scripts
(one for mask and one for auto), do it all with one script.
Receive Packet Steering is supported in two ways.
If 'auto' is used, then both threads on HT system will be
used for receive processing.
If explicit mask is given, then two masks can be used to set
both IRQ cpus and RPS cpus.
-rw-r--r-- | Makefile.am | 3 | ||||
-rwxr-xr-x | scripts/system/irq-affinity.pl | 289 | ||||
-rwxr-xr-x | scripts/system/vyatta-auto-irqaffin.pl | 467 | ||||
-rwxr-xr-x | scripts/system/vyatta-irqaffin | 194 | ||||
-rw-r--r-- | templates/interfaces/ethernet/node.tag/smp_affinity/node.def | 18 |
5 files changed, 294 insertions, 677 deletions
diff --git a/Makefile.am b/Makefile.am index d9c35d8a..f349121e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,8 +32,7 @@ sbin_SCRIPTS += scripts/system/vyatta_update_syslog.pl sbin_SCRIPTS += scripts/system/vyatta_update_console.pl sbin_SCRIPTS += scripts/system/vyatta_update_ntp.pl sbin_SCRIPTS += scripts/system/vyatta_update_telnet -sbin_SCRIPTS += scripts/system/vyatta-auto-irqaffin.pl -sbin_SCRIPTS += scripts/system/vyatta-irqaffin +sbin_SCRIPTS += scripts/system/irq-affinity.pl sbin_SCRIPTS += scripts/snmp/vyatta-snmp.pl sbin_SCRIPTS += scripts/snmp/snmpd.init sbin_SCRIPTS += scripts/snmp/if-mib-alias diff --git a/scripts/system/irq-affinity.pl b/scripts/system/irq-affinity.pl new file mode 100755 index 00000000..eaad95cc --- /dev/null +++ b/scripts/system/irq-affinity.pl @@ -0,0 +1,289 @@ +#!/usr/bin/perl + +# **** License **** +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# This code was originally developed by Vyatta, Inc. +# Portions created by Vyatta are Copyright (C) 2009,2010 Vyatta, Inc. +# All Rights Reserved. +# +# **** End License **** +# +use warnings; +use strict; +use Sys::Syslog qw(:standard :macros); + +die "Usage: $0 ifname {auto | mask}\n" if ($#ARGV < 1); + +my ($ifname, $mask) = @ARGV; + +die "Error: Interface $ifname does not exist\n" + unless -d "/sys/class/net/$ifname"; + +openlog("irq-affinity","",LOG_LOCAL0); + +my ( $cpus, $cores ) = cpuinfo(); + +if ($mask eq 'auto') { + affinity_auto($ifname); +} else { + affinity_mask($ifname, $mask); +} + +exit 0; + +# Get current irq assignments by reading /proc/interrupts +sub irqinfo { + my $irqmap; + + open( my $f, '<', "/proc/interrupts" ) + or die "Can't read /proc/interrupts"; + + while (<$f>) { + chomp; + my @cols = split; + + # First column is IRQ number (and colon) + next unless /^\s*(\d+):\s/; + my $irq = $1; + + # Skip columns for IRQ's per CPU + foreach my $name ( @cols[ $cpus+1 .. $#cols ] ) { + $name =~ s/,$//; + $irqmap->{$name} = $irq; + } + } + close $f; + + return $irqmap; +} + +# Determine number of cpus and cores +sub cpuinfo { + my ( $cpu, $core ); + + open( my $f, '<', "/proc/cpuinfo" ) + or die "Can't read /proc/cpuinfo"; + + while (<$f>) { + chomp; + if (/^cpu cores\s+:\s(\d+)$/) { + $core = $1; + } + elsif (/^processor\s+:\s+(\d)$/) { + $cpu = $1; + } + } + close $f; + + return ( $cpu + 1, $core ); +} + +# Set affinity value for a irq +sub set_affinity { + my ( $ifname, $irq, $mask ) = @_; + my $smp_affinity = "/proc/irq/$irq/smp_affinity"; + + syslog(LOG_INFO, "%s: irq %d affinity set to 0x%x", $ifname, $irq, $mask); + + open( my $f, '>', $smp_affinity ) + or die "Can't open: $smp_affinity : $!\n"; + printf {$f} "%x\n", $mask; + close $f; +} + +# set Receive Packet Steering mask +sub set_rps { + my ( $ifname, $q, $mask ) = @_; + + # ignore if older kernel without RPS + my $rxq = "/sys/class/net/$ifname/queues"; + return unless ( -d $rxq ); + + syslog(LOG_INFO, "%s: receive queue %d cpus set to 0x%x", + $ifname, $q, $mask); + + my $rps_cpus = "$rxq/rx-$q/rps_cpus"; + open( my $f, '>', $rps_cpus ) + or die "Can't open: $rps_cpus : $!\n"; + printf {$f} "%x\n", $mask; + close $f; +} + +# For multi-queue NIC choose next cpu to be on next core +# FIXME assumes all cpu's online +sub next_cpu { + my $cpu = shift; + my $threads = $cpus / $cores; # threads per core + + $cpu += $threads; + if ( $cpu >= $cpus ) { + $cpu = ($cpu + 1) % $threads; # next thread on core 0 + } + return $cpu; +} + +# First cpu to assign for the queues +sub first_cpu { + my ($ifname, $numq) = @_; + + # For multi-queue nic's always starts with 0 + # This is less than ideal when there are more core's available + # than number of queues (probably should barber pole); + # but the Intel IXGBE needs CPU 0 <-> queue 0 because of flow director + return 0 if ($numq > 1); + + # For single-queue nic choose IRQ based on name + # Ideally should make decision on least loaded CPU + my ($ifunit) = ($ifname =~ m/^[a-z]*(\d+)$/); + die "can't find number for $ifname\n" + unless defined($ifunit); + + return ( $ifunit * ($cpus / $cores) ) % $cpus; +} + +# Assignment for multi-queue NICs +# Assign each queue to successive cores +sub assign_multiqueue { + my ( $ifname, $numq, $irqmap, $irqfmt ) = @_; + my $cpu = 0; + + for ( my $q = first_cpu($ifname, $numq) ; $q < $numq ; $q++ ) { + # handles multiple irq's per interface (tx/rx) + foreach my $fmt (@$irqfmt) { + my $name = sprintf( $fmt, $ifname, $q ); + my $irq = $irqmap->{$name}; + + syslog(LOG_INFO, "%s: queue %d assign %s to cpu %d", + $ifname, $q, $name, $cpu ); + + # Assign CPU affinity for both IRQs + set_affinity( $ifname, $irq, 1 << $cpu ); + + # TODO use RPS to steer data if cores > queues? + } + $cpu = next_cpu($cpu); + } +} + +# Affinity assignment function for single-queue NICs. The strategy +# here is to just spread the interrupts of different NICs evenly +# across all CPUs. That is the best we can do without monitoring the +# load and traffic patterns. So we just directly map the NIC unit +# number into a CPU number. +sub assign_single { + my ( $ifname, $irq ) = @_; + my $cpu = first_cpu($ifname, 1); + + syslog( LOG_INFO, "%s: assign irq %d to cpu %d", $ifname, $irq, $cpu ); + + set_affinity( $ifname, $irq, 1 << $cpu ); + + my $threads = $cpus / $cores; + if ($threads > 1) { + # Use both threads on this cpu if hyperthreading + my $mask = ((1 << $threads) - 1) << $cpu; + set_rps($ifname, 0, $mask); + } + # MAYBE - Use all cpu's if no HT +} + +# find irq number used for given interface using sysfs +sub get_irq { + my $ifname = shift; + + open( my $irqf, '<', "/sys/class/net/$ifname/device/irq" ) + or warn "$ifname: can't find irq : $!\n"; + my $irq = <$irqf>; + chomp $irq; + close $irqf; + + return $irq; +} + +# Mask must contain at least one CPU and +# no bits outside of range of available CPU's +sub check_mask { + my ($ifname, $name, $mask) = @_; + my $m = hex($mask); + + die "$ifname: $name mask $mask has no bits set\n" + if ($m == 0); + + die "$ifname: $name mask $mask to large for number of CPU's: $cpus\n" + if ($m >= 1 << $cpus); +} + +# Set affinity (and RPS) based on mask +sub affinity_mask { + my ($ifname, $mask) = @_; + + # match on <hex> or <hex>,<hex> + unless ($mask =~ /^([0-9a-f]+)(|,([0-9a-f]+))$/) { + die "$ifname: irq mask $mask is not a valid affinity mask\n" + } + + my $irq = $1; + my $rps = $3; + + check_mask($ifname, "irq", $irq); + check_mask($ifname, "rps", $rps) if $rps; + + set_affinity($ifname, get_irq($ifname), hex($irq)); + set_rps($ifname, 0, hex($rps)) if $rps; +} + +# The auto strategy involves trying to achieve the following goals: +# +# - Spread the receive load among as many CPUs as possible. +# +# - For all multi-queue NICs in the system that provide both tx and +# rx queues, keep all of the queues that share the same queue +# number on same CPUs. I.e. tx and rx queue 0 of all such NICs +# should interrupt one CPU; tx and rx queue 1 should interrupt a +# different CPU, etc. +# +# - If hyperthreading is supported and enabled, avoid assigning +# queues to both CPUs of a hyperthreaded pair if there are enough +# CPUs available to do that. +sub affinity_auto { + my $ifname = shift; + my $irqmap = irqinfo(); + my @irqnames = keys %{$irqmap}; + + # Figure out what style of irq naming is being used + my $numirq = grep { /$ifname/ } @irqnames; + if ( $numirq > 1 ) { + my $nq = grep { /$ifname-rx-/ } @irqnames; + + if ( $nq > 0 ) { + my $ntx = grep { /$ifname-tx-/ } @irqnames; + die "$ifname: rx queues $nq != tx queues $ntx" + unless ( $nq == $ntx ); + + return assign_multiqueue( $ifname, $nq, $irqmap, + [ '%s-rx-%d', '%s-tx-%d' ] ); + } + + $nq = grep { /$ifname-TxRx-/ } @irqnames; + if ( $nq > 0 ) { + return assign_multiqueue( $ifname, $nq, $irqmap, ['%s-TxRx-%d'] ); + } + + $nq = grep { /$ifname-\d$/ } @irqnames; + if ( $nq > 0 ) { + return assign_multiqueue( $ifname, $nq, $irqmap, ['%s-%d'] ); + } + + die "Unknown multiqueue device naming for $ifname\n"; + } + + assign_single( $ifname, get_irq($ifname) ); +} diff --git a/scripts/system/vyatta-auto-irqaffin.pl b/scripts/system/vyatta-auto-irqaffin.pl deleted file mode 100755 index 54b75638..00000000 --- a/scripts/system/vyatta-auto-irqaffin.pl +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/perl -# -# Module: vyatta-auto-irqaffin.pl -# -# **** License **** -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# This code was originally developed by Vyatta, Inc. -# Portions created by Vyatta are Copyright (C) 2009,2010 Vyatta, Inc. -# All Rights Reserved. -# -# Author: Bob Gilligan (gilligan@vyatta.com) -# Date: October 2009 -# Description: Script to configure optimal IRQ affinity for NICs. -# -# **** End License **** -# - -# This script attempts to set up a static CPU affinity for the IRQs -# used by network interfaces. It is primarily targeted at supporting -# multi-queue NICs, but does include code to handle single-queue NICs. -# Since different NICs may have different queue organizations, and -# because there is no standard API for learning the mapping between -# queues and IRQ numbers, different code is required for each of the -# queue naming conventions. -# -# The general strategy involves trying to achieve the following goals: -# -# - Spread the receive load among as many CPUs as possible. -# -# - For all multi-queue NICs in the system that provide both tx and -# rx queues, keep all of the queues that share the same queue -# number on same CPUs. I.e. tx and rx queue 0 of all such NICs -# should interrupt one CPU; tx and rx queue 1 should interrupt a -# different CPU, etc. -# -# - If hyperthreading is supported and enabled, avoid assigning -# queues to both CPUs of a hyperthreaded pair if there are enough -# CPUs available to do that. -# -# This strategy yields the greatest MP scaling possible for -# multi-queue NICs. It also ensures that an individual skb is -# processed on the same CPU for the entirity of its lifecycle, -# including transmit time, which optimally utilizes the cache and -# keeps performance high. -# - - -use lib "/opt/vyatta/share/perl5"; -use Getopt::Long; - -use warnings; -use strict; - -# Send output of shell commands to syslog for debugging and so that -# the user is not confused by it. Log at debug level, which is supressed -# by default, so that we don't unnecessarily fill up the syslog file. -my $logger = 'logger -t firewall-cfg -p local0.debug --'; - -# Enable printing debug output to stdout. -my $debug_flag = 0; -my $syslog_flag = 0; - -my $setup_ifname; - -GetOptions("setup=s" => \$setup_ifname, - "debug" => \$debug_flag - ); - -sub log_msg { - my $message = shift; - - print "DEBUG: $message" if $debug_flag; - system("$logger DEBUG: \"$message\"") if $syslog_flag; -} - - -# Affinity assignment function for the Intel igb, ixgb and ixgbe -# drivers, and any other NICs that follow their queue naming -# convention. These NICs have an equal number of rx and tx queues. -# The first part of the strategy for optimal performance is to select -# the CPU to assign the IRQs to by mapping from the queue number. -# This ensures that all queues with the same queue number are assigned -# to the same CPU. The second part is to avoid assigning any queues -# to the second CPU in a hyper-threaded pair, if posible. I.e., if -# CPU 0 and 1 are hyper-threaded pairs, then assign a queue to CPU 0, -# but try to avoid assigning one to to CPU 1. But if we have more -# queues than CPUs, then it is OK to assign some to the second CPU in -# a hyperthreaded pair. -# -sub intel_func{ - my ($ifname, $numcpus, $numcores) = @_; - my $rx_queues; # number of rx queues - my $tx_queues; # number of tx queues - my $ht_factor; # 2 if HT enabled, 1 if not - my $start_cpu; # CPU number to start assignment at - - log_msg("intel_func was called.\n"); - - if ($numcpus > $numcores) { - $ht_factor = 2; - } else { - $ht_factor = 1; - } - - log_msg("ht_factor is $ht_factor.\n"); - - # Figure out how many queues we have - - $rx_queues=`grep "$ifname-rx-" /proc/interrupts | wc -l`; - $rx_queues =~ s/\n//; - - $tx_queues=`grep "$ifname-tx-" /proc/interrupts | wc -l`; - $tx_queues =~ s/\n//; - - log_msg("rx_queues is $rx_queues. tx_queues is $tx_queues\n"); - - if ($rx_queues != $tx_queues) { - printf("Error: rx and tx queues don't match for igb driver.\n"); - exit 1; - } - - - # Special case of a single-queue masquarading as a multi-queue NIC - if ($rx_queues == 1) { - $ifname =~ m/^eth(.*)$/; - - my $ifunit = $1; - log_msg ("ifunit = $ifunit\n"); - - if ($numcpus > $numcores) { - # Hyperthreaded - $start_cpu = (2 * $ifunit) % $numcpus; - - # every other time it wraps, add one to use the hyper-thread pair - # of the CPU selected. - my $use_ht = ((2 * $ifunit) / $numcpus) % 2; - $start_cpu += $use_ht; - } else { - # Not hyperthreaded. Map it to unit number MOD number of linux CPUs. - $start_cpu = $ifunit % $numcpus; - } - } else { - $start_cpu = 0; - } - - # For i = 0 to number of queues: - # Affinity of rx and tx queue $i gets CPU ($i * (2 if HT, 1 if no HT)) - # % number_of_cpus - for (my $queue = 0, my $cpu = $start_cpu; ($queue < $rx_queues) ; - $queue++) { - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("queue=$queue cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Get the IRQ number for RX queue - my $rx_irq=`grep "$ifname-rx-$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $rx_irq =~ s/\n//; - $rx_irq =~ s/ //g; - - # Get the IRQ number for TX queue - my $tx_irq=`grep "$ifname-tx-$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $tx_irq =~ s/\n//; - $tx_irq =~ s/ //g; - - log_msg("rx_irq = $rx_irq. tx_irq = $tx_irq\n"); - - # Assign CPU affinity for both IRQs - system "echo $cpu_hex > /proc/irq/$rx_irq/smp_affinity"; - system "echo $cpu_hex > /proc/irq/$tx_irq/smp_affinity"; - - $cpu += $ht_factor; - - if ($cpu >= $numcpus) { - # Must "wrap" - $cpu %= $numcpus; - - if ($ht_factor > 1) { - # Next time through, select the other CPU in a hyperthreaded - # pair. - if ($cpu == 0) { - $cpu++; - } else { - $cpu--; - } - } - } - } -}; - - -# Affinity setting function for NICs using new intel queue scheme -# that provides one IRQ for each pair of TX and RX queues -sub intel_new_func{ - my ($ifname, $numcpus, $numcores) = @_; - my $txrx_queues; # number of rx/rx queue pairs - my $ht_factor; # 2 if HT enabled, 1 if not - - log_msg("intel_new_func was called.\n"); - - if ($numcpus > $numcores) { - $ht_factor = 2; - } else { - $ht_factor = 1; - } - - log_msg("ht_factor is $ht_factor.\n"); - - # Figure out how many queues we have - - $txrx_queues=`grep "$ifname-TxRx-" /proc/interrupts | wc -l`; - $txrx_queues =~ s/\n//; - - log_msg("txrx_queues is $txrx_queues.\n"); - - if ($txrx_queues <= 0) { - printf("Error: No TxRx queues found for new intel driver.\n"); - exit 1; - } - - # For i = 0 to number of queues: - # Affinity of TX/RX queue $i gets CPU ($i * (2 if HT, 1 if no HT)) - # % number_of_cpus - for (my $queue = 0, my $cpu = 0; ($queue < $txrx_queues) ; $queue++) { - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("queue=$queue cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Get the IRQ number for RX queue - my $txrx_irq=`grep "$ifname-TxRx-$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $txrx_irq =~ s/\n//; - $txrx_irq =~ s/ //g; - - log_msg("txrx_irq = $txrx_irq.\n"); - - # Assign CPU affinity for this IRQs - system "echo $cpu_hex > /proc/irq/$txrx_irq/smp_affinity"; - - $cpu += $ht_factor; - - if ($cpu >= $numcpus) { - # Must "wrap" - $cpu %= $numcpus; - - if ($ht_factor > 1) { - # Next time through, select the other CPU in a hyperthreaded - # pair. - if ($cpu == 0) { - $cpu++; - } else { - $cpu--; - } - } - } - } -}; - - -# Affinity assignment function for Broadcom NICs using the bnx2 driver -# or other multi-queue NICs that follow their queue naming convention. -# This strategy is similar to that for Intel drivers. But since -# Broadcom NICs do not have separate receive and transmit queues we -# perform one affinity assignment per queue. -# -sub broadcom_func{ - my ($ifname, $numcpus, $numcores) = @_; - my $num_queues; # number of queues - my $ht_factor; # 2 if HT enabled, 1 if not - - log_msg("broadcom_func was called.\n"); - - # Figure out how many queues we have - $num_queues=`egrep "$ifname\[-.\]\{1\}" /proc/interrupts | wc -l`; - $num_queues =~ s/\n//; - - log_msg("num_queues=$num_queues\n"); - - if ($num_queues <=0) { - printf("ERROR: No queues found for $ifname\n"); - exit 1; - } - - if ($numcpus > $numcores) { - $ht_factor = 2; - } else { - $ht_factor = 1; - } - - log_msg("ht_factor is $ht_factor.\n"); - - for (my $queue = 0, my $cpu = 0; ($queue < $num_queues) ; $queue++) { - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("queue=$queue cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Get the IRQ number for the queue - my $irq=`egrep "$ifname\[-.fp\]*$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $irq =~ s/\n//; - $irq =~ s/ //g; - - log_msg("irq = $irq.\n"); - - # Assign CPU affinity for this IRQs - system "echo $cpu_hex > /proc/irq/$irq/smp_affinity"; - - $cpu += $ht_factor; - if ($cpu >= $numcpus) { - # Must "wrap" - $cpu %= $numcpus; - - if ($ht_factor > 1) { - # Next time through, select the other CPU in a hyperthreaded - # pair. - if ($cpu == 0) { - $cpu++; - } else { - $cpu--; - } - } - } - } -} - - -# Affinity assignment function for single-quque NICs. The strategy -# here is to just spread the interrupts of different NICs evenly -# across all CPUs. That is the best we can do without monitoring the -# load and traffic patterns. So we just directly map the NIC unit -# number into a CPU number. -# -sub single_func { - my ($ifname, $numcpus, $numcores) = @_; - my $cpu; - use integer; - - log_msg("single_func was calledn.\n"); - - $ifname =~ m/^eth(.*)$/; - - my $ifunit = $1; - log_msg ("ifunit = $ifunit\n"); - - # Get the IRQ number for the queue - my $irq=`grep "$ifname" /proc/interrupts | awk -F: '{print \$1}'`; - $irq =~ s/\n//; - $irq =~ s/ //g; - - log_msg("irq = $irq.\n"); - - # Figure out what CPU to assign it to - if ($numcpus > $numcores) { - # Hyperthreaded - $cpu = (2 * $ifunit) % $numcpus; - - # every other time it wraps, add one to use the hyper-thread pair - # of the CPU selected. - my $use_ht = ((2 * $ifunit) / $numcpus) % 2; - $cpu += $use_ht; - } else { - # Not hyperthreaded. Map it to unit number MOD number of linux CPUs. - $cpu = $ifunit % $numcpus; - } - - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Assign CPU affinity for this IRQs - system "echo $cpu_hex > /proc/irq/$irq/smp_affinity"; -} - -# Mapping from driver type to function that handles it. -my %driver_hash = ( 'intel' => \&intel_func, - 'intel_new' => \&intel_new_func, - 'broadcom' => \&broadcom_func, - 'single' => \&single_func); - -if (defined $setup_ifname) { - # Set up automatic IRQ affinity for the named interface - - log_msg("setup $setup_ifname\n"); - - my $ifname = $setup_ifname; # shorter variable name - my $drivername; # Name of the NIC driver, e.g. "igb". - my $numcpus; # Number of Linux "cpus" - my $numcores; # Number of unique CPU cores - my $driver_func; # Pointer to fuction specific to a driver - my $driver_style; # Style of the driver. Whether it is multi-queue - # or not, and if it is, how it names its queues. - - # Determine how many CPUs the machine has. - $numcpus=`grep "^processor" /proc/cpuinfo | wc -l`; - $numcpus =~ s/\n//; - - log_msg("numcpus is $numcpus\n"); - - if ($numcpus == 1) { - # Nothing to do if we only have one CPU, so just exit quietly. - exit 0; - } - - # Determine how many cores the machine has. Could be less than - # the number of CPUs if processor supports hyperthreading. - $numcores=`grep "^core id" /proc/cpuinfo | uniq | wc -l`; - $numcores =~ s/\n//; - - log_msg("numcores is $numcores.\n"); - - # Verify that interface exists - if (! (-e "/proc/sys/net/ipv4/conf/$ifname")) { - printf("Error: Interface $ifname does not exist\n"); - exit 1; - } - - # Figure out what style of driver this NIC is using. - my $numints=`grep $ifname /proc/interrupts | wc -l`; - $numints =~ s/\n//; - if ($numints > 1) { - # It is a multiqueue NIC. Now figure out which one. - my $rx_queues=`grep "$ifname-rx-" /proc/interrupts | wc -l`; - $rx_queues =~ s/\n//; - if ($rx_queues > 0) { - # Driver is following the original Intel queue naming style - $driver_style="intel"; - } else { - my $rx_queues=`grep "$ifname-TxRx-" /proc/interrupts | wc -l`; - if ($rx_queues > 0) { - # Driver is following the new Intel queue naming - # style where on IRQ is used for each pair of - # TX and RX queues - $driver_style="intel_new"; - } else { - # The only other queue naming style that we have seen is the - # one used by Broadcom NICs. - $driver_style="broadcom"; - } - } - } elsif ($numints == 1) { - # It is a single queue NIC. - $driver_style="single"; - } else { - # $numints must be 0 - printf("Unable to determine IRQs for interface $ifname.\n"); - exit 0; - } - $driver_func = $driver_hash{$driver_style}; - - &$driver_func($ifname, $numcpus, $numcores); - - exit 0; -} - -printf("Must specify options.\n"); -exit(1); - - diff --git a/scripts/system/vyatta-irqaffin b/scripts/system/vyatta-irqaffin deleted file mode 100755 index 092da818..00000000 --- a/scripts/system/vyatta-irqaffin +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash - -# Author: Robert E. Gilligan <gilligan@vyatta.com> -# Date: 2008 -# Description: CLI back-end script to manipulate NIC interrupt CPU affinity. - -# **** License **** -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# This code was originally developed by Vyatta, Inc. -# Portions created by Vyatta are Copyright (C) 2006, 2007, 2008 Vyatta, Inc. -# All Rights Reserved. -# **** End License **** - -# Provides sub-commands to: -# - Check the validity of an affinity mask value -# - Set the affinity mask to the IRQs being used by an interface -# - Reset the affinity mask of the IRQs being used by an interface to the -# system default value of all-ones. -# - Print the affinity mask of the IRQs being used by an interface -# -# If the NIC in question supports multiple IRQs, the "set" sub-command -# sets all IRQs to the same mask. The "print" sub-command displays -# the mask of each IRQ individually. -# - -# Max number of hex characters in an IRQ affinity mask. Support up to 64 CPUs. -MAX_MASK=16 - -# Set up some global values... -numcpus=`grep -c -e "^processor" /proc/cpuinfo` -declare -i maxmask=(2**numcpus) -let maxmask=maxmask-1 -maxmaskhex=`printf "%x" ${maxmask}` - -print_usage() -{ - echo "Usage:" - echo -e "\t$0 check <ifname> <mask>" - echo -e "\t$0 set <ifname> <mask>" - echo -e "\t$0 reset <ifname>" - echo -e "\t$0 print <ifname>" -} - -get_irqnums() -{ - irqnums=`grep $1 /proc/interrupts | awk -F ': ' '{ print $1 }'` - if [ -z "$irqnums" ]; then - echo "Unable to determine IRQs for interface $1" - return 1 - fi - return 0 -} - - -get_mask() -{ - mask=$1 - - # mask must be a short hex value - if [ ${#mask} -gt $MAX_MASK ]; then - echo "mask too long: ${#2} characters." - return 1 - fi - - # strip out all the hex digits - exmask=`echo $mask | sed -e s/[0-9a-fA-F]//g` - - # if anything is left, its not hex - if [ ! -z "$exmask" ]; then - echo "Invalid characters in hex mask: $exmask" - return 1 - fi - - declare -i intmask=0x${mask} - - # Make sure that mask holds at least one bit, and holds no more bits - # than we have CPUs. - - if [ ${intmask} -eq 0 ]; then - echo "Mask can not be 0." - return 1 - fi - - if [ $intmask -gt $maxmask ]; then - echo "Mask is too large. Maximum hexidecimal bitmask is: ${maxmaskhex}" - return 1 - fi - - return 0 -} - - -case "$1" in - check) - # Note: We don't validate the interface name even though - # it is available as a command argument. That is because - # the interface may not exist or may not be configured at - # the time the check is performed. - # - if [ $# -ne 3 ]; then - print_usage - exit 1 - fi - - if ! get_mask $3 ; then - exit 1 - fi - exit 0 - ;; - - set) - if [ $# -ne 3 ]; then - print_usage - exit 1 - fi - - if ! get_irqnums $2 ; then - exit 1 - fi - - if ! get_mask $3 ; then - exit 1 - fi - - for irqnum in $irqnums ; do - echo $mask > /proc/irq/$irqnum/smp_affinity - done - - if [ $? -ne 0 ]; then - echo "Couldn't assign smp_affinity. Exit status: $?" - exit 1 - fi - ;; - - reset) - if [ $# -ne 2 ]; then - print_usage - exit 1 - fi - if ! get_irqnums $2 ; then - exit 1 - fi - - if [ -e /proc/irq/default_smp_affinity ]; then - defmask=`cat /proc/irq/default_smp_affinity` - else - defmask=$maxmaskhex - fi - - for irqnum in $irqnums ; do - echo $defmask > /proc/irq/$irqnum/smp_affinity - if [ $? -ne 0 ]; then - echo "Couldn't assign smp_affinity for IRQ $irqnum. Exit status: $?" - exit 1 - fi - done - ;; - - - print) - if [ $# -ne 2 ]; then - print_usage - exit 1 - fi - if ! get_irqnums $2 ; then - exit 1 - fi - - for irqnum in $irqnums ; do - mask=`cat /proc/irq/$irqnum/smp_affinity` - - if [ -z $mask ]; then - echo "Couldn't get smp_affinity for interface $2, irq $irqnum" - exit 1 - fi - - echo "Interface: $2 IRQ: $irqnum Mask: $mask" - done - ;; - - *) - print_usage - exit 1 - ;; - -esac diff --git a/templates/interfaces/ethernet/node.tag/smp_affinity/node.def b/templates/interfaces/ethernet/node.tag/smp_affinity/node.def index 671a28f0..1f8a2610 100644 --- a/templates/interfaces/ethernet/node.tag/smp_affinity/node.def +++ b/templates/interfaces/ethernet/node.tag/smp_affinity/node.def @@ -15,24 +15,14 @@ type: txt help: CPU interrupt affinity mask -val_help: <hex>; Bitmask representing CPUs that this NIC will interrupt val_help: auto; Automatic CPU affinity (default) +val_help: <hex>; Bitmask representing CPUs that this NIC will interrupt +val_help: <hex>,<hex>; Bitmasks representing CPUs for interrupt and receive processing default: "auto" -syntax:expression: $VAR(@) == "auto" || pattern $VAR(@) "^[0-9a-f]+$" ; +syntax:expression: $VAR(@) == "auto" || pattern $VAR(@) "^[0-9a-f]+(|,[0-9a-f]+)$" ; "IRQ affinity mask must be hex value or auto" -commit:expression: exec "\ - [ \"$VAR(@)\" == \"auto\" ] || \ - /opt/vyatta/sbin/vyatta-irqaffin check $VAR(../@) $VAR(@)" - -update: if [ "$VAR(@)" = "auto" ]; then - sudo /opt/vyatta/sbin/vyatta-auto-irqaffin.pl --setup $VAR(../@) - else - sudo /opt/vyatta/sbin/vyatta-irqaffin set $VAR(../@) $VAR(@) - fi - -delete: [ -d /sys/class/net/$VAR(../@) ] || exit 0 - sudo /opt/vyatta/sbin/vyatta-irqaffin reset $VAR(../@) +update: sudo /opt/vyatta/sbin/irq-affinity.pl $VAR(../@) $VAR(@) |