patch-realtime 753 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590135911359213593135941359513596135971359813599136001360113602136031360413605136061360713608136091361013611136121361313614136151361613617136181361913620136211362213623136241362513626136271362813629136301363113632136331363413635136361363713638136391364013641136421364313644136451364613647136481364913650136511365213653136541365513656136571365813659136601366113662136631366413665136661366713668136691367013671136721367313674136751367613677136781367913680136811368213683136841368513686136871368813689136901369113692136931369413695136961369713698136991370013701137021370313704137051370613707137081370913710137111371213713137141371513716137171371813719137201372113722137231372413725137261372713728137291373013731137321373313734137351373613737137381373913740137411374213743137441374513746137471374813749137501375113752137531375413755137561375713758137591376013761137621376313764137651376613767137681376913770137711377213773137741377513776137771377813779137801378113782137831378413785137861378713788137891379013791137921379313794137951379613797137981379913800138011380213803138041380513806138071380813809138101381113812138131381413815138161381713818138191382013821138221382313824138251382613827138281382913830138311383213833138341383513836138371383813839138401384113842138431384413845138461384713848138491385013851138521385313854138551385613857138581385913860138611386213863138641386513866138671386813869138701387113872138731387413875138761387713878138791388013881138821388313884138851388613887138881388913890138911389213893138941389513896138971389813899139001390113902139031390413905139061390713908139091391013911139121391313914139151391613917139181391913920139211392213923139241392513926139271392813929139301393113932139331393413935139361393713938139391394013941139421394313944139451394613947139481394913950139511395213953139541395513956139571395813959139601396113962139631396413965139661396713968139691397013971139721397313974139751397613977139781397913980139811398213983139841398513986139871398813989139901399113992139931399413995139961399713998139991400014001140021400314004140051400614007140081400914010140111401214013140141401514016140171401814019140201402114022140231402414025140261402714028140291403014031140321403314034140351403614037140381403914040140411404214043140441404514046140471404814049140501405114052140531405414055140561405714058140591406014061140621406314064140651406614067140681406914070140711407214073140741407514076140771407814079140801408114082140831408414085140861408714088140891409014091140921409314094140951409614097140981409914100141011410214103141041410514106141071410814109141101411114112141131411414115141161411714118141191412014121141221412314124141251412614127141281412914130141311413214133141341413514136141371413814139141401414114142141431414414145141461414714148141491415014151141521415314154141551415614157141581415914160141611416214163141641416514166141671416814169141701417114172141731417414175141761417714178141791418014181141821418314184141851418614187141881418914190141911419214193141941419514196141971419814199142001420114202142031420414205142061420714208142091421014211142121421314214142151421614217142181421914220142211422214223142241422514226142271422814229142301423114232142331423414235142361423714238142391424014241142421424314244142451424614247142481424914250142511425214253142541425514256142571425814259142601426114262142631426414265142661426714268142691427014271142721427314274142751427614277142781427914280142811428214283142841428514286142871428814289142901429114292142931429414295142961429714298142991430014301143021430314304143051430614307143081430914310143111431214313143141431514316143171431814319143201432114322143231432414325143261432714328143291433014331143321433314334143351433614337143381433914340143411434214343143441434514346143471434814349143501435114352143531435414355143561435714358143591436014361143621436314364143651436614367143681436914370143711437214373143741437514376143771437814379143801438114382143831438414385143861438714388143891439014391143921439314394143951439614397143981439914400144011440214403144041440514406144071440814409144101441114412144131441414415144161441714418144191442014421144221442314424144251442614427144281442914430144311443214433144341443514436144371443814439144401444114442144431444414445144461444714448144491445014451144521445314454144551445614457144581445914460144611446214463144641446514466144671446814469144701447114472144731447414475144761447714478144791448014481144821448314484144851448614487144881448914490144911449214493144941449514496144971449814499145001450114502145031450414505145061450714508145091451014511145121451314514145151451614517145181451914520145211452214523145241452514526145271452814529145301453114532145331453414535145361453714538145391454014541145421454314544145451454614547145481454914550145511455214553145541455514556145571455814559145601456114562145631456414565145661456714568145691457014571145721457314574145751457614577145781457914580145811458214583145841458514586145871458814589145901459114592145931459414595145961459714598145991460014601146021460314604146051460614607146081460914610146111461214613146141461514616146171461814619146201462114622146231462414625146261462714628146291463014631146321463314634146351463614637146381463914640146411464214643146441464514646146471464814649146501465114652146531465414655146561465714658146591466014661146621466314664146651466614667146681466914670146711467214673146741467514676146771467814679146801468114682146831468414685146861468714688146891469014691146921469314694146951469614697146981469914700147011470214703147041470514706147071470814709147101471114712147131471414715147161471714718147191472014721147221472314724147251472614727147281472914730147311473214733147341473514736147371473814739147401474114742147431474414745147461474714748147491475014751147521475314754147551475614757147581475914760147611476214763147641476514766147671476814769147701477114772147731477414775147761477714778147791478014781147821478314784147851478614787147881478914790147911479214793147941479514796147971479814799148001480114802148031480414805148061480714808148091481014811148121481314814148151481614817148181481914820148211482214823148241482514826148271482814829148301483114832148331483414835148361483714838148391484014841148421484314844148451484614847148481484914850148511485214853148541485514856148571485814859148601486114862148631486414865148661486714868148691487014871148721487314874148751487614877148781487914880148811488214883148841488514886148871488814889148901489114892148931489414895148961489714898148991490014901149021490314904149051490614907149081490914910149111491214913149141491514916149171491814919149201492114922149231492414925149261492714928149291493014931149321493314934149351493614937149381493914940149411494214943149441494514946149471494814949149501495114952149531495414955149561495714958149591496014961149621496314964149651496614967149681496914970149711497214973149741497514976149771497814979149801498114982149831498414985149861498714988149891499014991149921499314994149951499614997149981499915000150011500215003150041500515006150071500815009150101501115012150131501415015150161501715018150191502015021150221502315024150251502615027150281502915030150311503215033150341503515036150371503815039150401504115042150431504415045150461504715048150491505015051150521505315054150551505615057150581505915060150611506215063150641506515066150671506815069150701507115072150731507415075150761507715078150791508015081150821508315084150851508615087150881508915090150911509215093150941509515096150971509815099151001510115102151031510415105151061510715108151091511015111151121511315114151151511615117151181511915120151211512215123151241512515126151271512815129151301513115132151331513415135151361513715138151391514015141151421514315144151451514615147151481514915150151511515215153151541515515156151571515815159151601516115162151631516415165151661516715168151691517015171151721517315174151751517615177151781517915180151811518215183151841518515186151871518815189151901519115192151931519415195151961519715198151991520015201152021520315204152051520615207152081520915210152111521215213152141521515216152171521815219152201522115222152231522415225152261522715228152291523015231152321523315234152351523615237152381523915240152411524215243152441524515246152471524815249152501525115252152531525415255152561525715258152591526015261152621526315264152651526615267152681526915270152711527215273152741527515276152771527815279152801528115282152831528415285152861528715288152891529015291152921529315294152951529615297152981529915300153011530215303153041530515306153071530815309153101531115312153131531415315153161531715318153191532015321153221532315324153251532615327153281532915330153311533215333153341533515336153371533815339153401534115342153431534415345153461534715348153491535015351153521535315354153551535615357153581535915360153611536215363153641536515366153671536815369153701537115372153731537415375153761537715378153791538015381153821538315384153851538615387153881538915390153911539215393153941539515396153971539815399154001540115402154031540415405154061540715408154091541015411154121541315414154151541615417154181541915420154211542215423154241542515426154271542815429154301543115432154331543415435154361543715438154391544015441154421544315444154451544615447154481544915450154511545215453154541545515456154571545815459154601546115462154631546415465154661546715468154691547015471154721547315474154751547615477154781547915480154811548215483154841548515486154871548815489154901549115492154931549415495154961549715498154991550015501155021550315504155051550615507155081550915510155111551215513155141551515516155171551815519155201552115522155231552415525155261552715528155291553015531155321553315534155351553615537155381553915540155411554215543155441554515546155471554815549155501555115552155531555415555155561555715558155591556015561155621556315564155651556615567155681556915570155711557215573155741557515576155771557815579155801558115582155831558415585155861558715588155891559015591155921559315594155951559615597155981559915600156011560215603156041560515606156071560815609156101561115612156131561415615156161561715618156191562015621156221562315624156251562615627156281562915630156311563215633156341563515636156371563815639156401564115642156431564415645156461564715648156491565015651156521565315654156551565615657156581565915660156611566215663156641566515666156671566815669156701567115672156731567415675156761567715678156791568015681156821568315684156851568615687156881568915690156911569215693156941569515696156971569815699157001570115702157031570415705157061570715708157091571015711157121571315714157151571615717157181571915720157211572215723157241572515726157271572815729157301573115732157331573415735157361573715738157391574015741157421574315744157451574615747157481574915750157511575215753157541575515756157571575815759157601576115762157631576415765157661576715768157691577015771157721577315774157751577615777157781577915780157811578215783157841578515786157871578815789157901579115792157931579415795157961579715798157991580015801158021580315804158051580615807158081580915810158111581215813158141581515816158171581815819158201582115822158231582415825158261582715828158291583015831158321583315834158351583615837158381583915840158411584215843158441584515846158471584815849158501585115852158531585415855158561585715858158591586015861158621586315864158651586615867158681586915870158711587215873158741587515876158771587815879158801588115882158831588415885158861588715888158891589015891158921589315894158951589615897158981589915900159011590215903159041590515906159071590815909159101591115912159131591415915159161591715918159191592015921159221592315924159251592615927159281592915930159311593215933159341593515936159371593815939159401594115942159431594415945159461594715948159491595015951159521595315954159551595615957159581595915960159611596215963159641596515966159671596815969159701597115972159731597415975159761597715978159791598015981159821598315984159851598615987159881598915990159911599215993159941599515996159971599815999160001600116002160031600416005160061600716008160091601016011160121601316014160151601616017160181601916020160211602216023160241602516026160271602816029160301603116032160331603416035160361603716038160391604016041160421604316044160451604616047160481604916050160511605216053160541605516056160571605816059160601606116062160631606416065160661606716068160691607016071160721607316074160751607616077160781607916080160811608216083160841608516086160871608816089160901609116092160931609416095160961609716098160991610016101161021610316104161051610616107161081610916110161111611216113161141611516116161171611816119161201612116122161231612416125161261612716128161291613016131161321613316134161351613616137161381613916140161411614216143161441614516146161471614816149161501615116152161531615416155161561615716158161591616016161161621616316164161651616616167161681616916170161711617216173161741617516176161771617816179161801618116182161831618416185161861618716188161891619016191161921619316194161951619616197161981619916200162011620216203162041620516206162071620816209162101621116212162131621416215162161621716218162191622016221162221622316224162251622616227162281622916230162311623216233162341623516236162371623816239162401624116242162431624416245162461624716248162491625016251162521625316254162551625616257162581625916260162611626216263162641626516266162671626816269162701627116272162731627416275162761627716278162791628016281162821628316284162851628616287162881628916290162911629216293162941629516296162971629816299163001630116302163031630416305163061630716308163091631016311163121631316314163151631616317163181631916320163211632216323163241632516326163271632816329163301633116332163331633416335163361633716338163391634016341163421634316344163451634616347163481634916350163511635216353163541635516356163571635816359163601636116362163631636416365163661636716368163691637016371163721637316374163751637616377163781637916380163811638216383163841638516386163871638816389163901639116392163931639416395163961639716398163991640016401164021640316404164051640616407164081640916410164111641216413164141641516416164171641816419164201642116422164231642416425164261642716428164291643016431164321643316434164351643616437164381643916440164411644216443164441644516446164471644816449164501645116452164531645416455164561645716458164591646016461164621646316464164651646616467164681646916470164711647216473164741647516476164771647816479164801648116482164831648416485164861648716488164891649016491164921649316494164951649616497164981649916500165011650216503165041650516506165071650816509165101651116512165131651416515165161651716518165191652016521165221652316524165251652616527165281652916530165311653216533165341653516536165371653816539165401654116542165431654416545165461654716548165491655016551165521655316554165551655616557165581655916560165611656216563165641656516566165671656816569165701657116572165731657416575165761657716578165791658016581165821658316584165851658616587165881658916590165911659216593165941659516596165971659816599166001660116602166031660416605166061660716608166091661016611166121661316614166151661616617166181661916620166211662216623166241662516626166271662816629166301663116632166331663416635166361663716638166391664016641166421664316644166451664616647166481664916650166511665216653166541665516656166571665816659166601666116662166631666416665166661666716668166691667016671166721667316674166751667616677166781667916680166811668216683166841668516686166871668816689166901669116692166931669416695166961669716698166991670016701167021670316704167051670616707167081670916710167111671216713167141671516716167171671816719167201672116722167231672416725167261672716728167291673016731167321673316734167351673616737167381673916740167411674216743167441674516746167471674816749167501675116752167531675416755167561675716758167591676016761167621676316764167651676616767167681676916770167711677216773167741677516776167771677816779167801678116782167831678416785167861678716788167891679016791167921679316794167951679616797167981679916800168011680216803168041680516806168071680816809168101681116812168131681416815168161681716818168191682016821168221682316824168251682616827168281682916830168311683216833168341683516836168371683816839168401684116842168431684416845168461684716848168491685016851168521685316854168551685616857168581685916860168611686216863168641686516866168671686816869168701687116872168731687416875168761687716878168791688016881168821688316884168851688616887168881688916890168911689216893168941689516896168971689816899169001690116902169031690416905169061690716908169091691016911169121691316914169151691616917169181691916920169211692216923169241692516926169271692816929169301693116932169331693416935169361693716938169391694016941169421694316944169451694616947169481694916950169511695216953169541695516956169571695816959169601696116962169631696416965169661696716968169691697016971169721697316974169751697616977169781697916980169811698216983169841698516986169871698816989169901699116992169931699416995169961699716998169991700017001170021700317004170051700617007170081700917010170111701217013170141701517016170171701817019170201702117022170231702417025170261702717028170291703017031170321703317034170351703617037170381703917040170411704217043170441704517046170471704817049170501705117052170531705417055170561705717058170591706017061170621706317064170651706617067170681706917070170711707217073170741707517076170771707817079170801708117082170831708417085170861708717088170891709017091170921709317094170951709617097170981709917100171011710217103171041710517106171071710817109171101711117112171131711417115171161711717118171191712017121171221712317124171251712617127171281712917130171311713217133171341713517136171371713817139171401714117142171431714417145171461714717148171491715017151171521715317154171551715617157171581715917160171611716217163171641716517166171671716817169171701717117172171731717417175171761717717178171791718017181171821718317184171851718617187171881718917190171911719217193171941719517196171971719817199172001720117202172031720417205172061720717208172091721017211172121721317214172151721617217172181721917220172211722217223172241722517226172271722817229172301723117232172331723417235172361723717238172391724017241172421724317244172451724617247172481724917250172511725217253172541725517256172571725817259172601726117262172631726417265172661726717268172691727017271172721727317274172751727617277172781727917280172811728217283172841728517286172871728817289172901729117292172931729417295172961729717298172991730017301173021730317304173051730617307173081730917310173111731217313173141731517316173171731817319173201732117322173231732417325173261732717328173291733017331173321733317334173351733617337173381733917340173411734217343173441734517346173471734817349173501735117352173531735417355173561735717358173591736017361173621736317364173651736617367173681736917370173711737217373173741737517376173771737817379173801738117382173831738417385173861738717388173891739017391173921739317394173951739617397173981739917400174011740217403174041740517406174071740817409174101741117412174131741417415174161741717418174191742017421174221742317424174251742617427174281742917430174311743217433174341743517436174371743817439174401744117442174431744417445174461744717448174491745017451174521745317454174551745617457174581745917460174611746217463174641746517466174671746817469174701747117472174731747417475174761747717478174791748017481174821748317484174851748617487174881748917490174911749217493174941749517496174971749817499175001750117502175031750417505175061750717508175091751017511175121751317514175151751617517175181751917520175211752217523175241752517526175271752817529175301753117532175331753417535175361753717538175391754017541175421754317544175451754617547175481754917550175511755217553175541755517556175571755817559175601756117562175631756417565175661756717568175691757017571175721757317574175751757617577175781757917580175811758217583175841758517586175871758817589175901759117592175931759417595175961759717598175991760017601176021760317604176051760617607176081760917610176111761217613176141761517616176171761817619176201762117622176231762417625176261762717628176291763017631176321763317634176351763617637176381763917640176411764217643176441764517646176471764817649176501765117652176531765417655176561765717658176591766017661176621766317664176651766617667176681766917670176711767217673176741767517676176771767817679176801768117682176831768417685176861768717688176891769017691176921769317694176951769617697176981769917700177011770217703177041770517706177071770817709177101771117712177131771417715177161771717718177191772017721177221772317724177251772617727177281772917730177311773217733177341773517736177371773817739177401774117742177431774417745177461774717748177491775017751177521775317754177551775617757177581775917760177611776217763177641776517766177671776817769177701777117772177731777417775177761777717778177791778017781177821778317784177851778617787177881778917790177911779217793177941779517796177971779817799178001780117802178031780417805178061780717808178091781017811178121781317814178151781617817178181781917820178211782217823178241782517826178271782817829178301783117832178331783417835178361783717838178391784017841178421784317844178451784617847178481784917850178511785217853178541785517856178571785817859178601786117862178631786417865178661786717868178691787017871178721787317874178751787617877178781787917880178811788217883178841788517886178871788817889178901789117892178931789417895178961789717898178991790017901179021790317904179051790617907179081790917910179111791217913179141791517916179171791817919179201792117922179231792417925179261792717928179291793017931179321793317934179351793617937179381793917940179411794217943179441794517946179471794817949179501795117952179531795417955179561795717958179591796017961179621796317964179651796617967179681796917970179711797217973179741797517976179771797817979179801798117982179831798417985179861798717988179891799017991179921799317994179951799617997179981799918000180011800218003180041800518006180071800818009180101801118012180131801418015180161801718018180191802018021180221802318024180251802618027180281802918030180311803218033180341803518036180371803818039180401804118042180431804418045180461804718048180491805018051180521805318054180551805618057180581805918060180611806218063180641806518066180671806818069180701807118072180731807418075180761807718078180791808018081180821808318084180851808618087180881808918090180911809218093180941809518096180971809818099181001810118102181031810418105181061810718108181091811018111181121811318114181151811618117181181811918120181211812218123181241812518126181271812818129181301813118132181331813418135181361813718138181391814018141181421814318144181451814618147181481814918150181511815218153181541815518156181571815818159181601816118162181631816418165181661816718168181691817018171181721817318174181751817618177181781817918180181811818218183181841818518186181871818818189181901819118192181931819418195181961819718198181991820018201182021820318204182051820618207182081820918210182111821218213182141821518216182171821818219182201822118222182231822418225182261822718228182291823018231182321823318234182351823618237182381823918240182411824218243182441824518246182471824818249182501825118252182531825418255182561825718258182591826018261182621826318264182651826618267182681826918270182711827218273182741827518276182771827818279182801828118282182831828418285182861828718288182891829018291182921829318294182951829618297182981829918300183011830218303183041830518306183071830818309183101831118312183131831418315183161831718318183191832018321183221832318324183251832618327183281832918330183311833218333183341833518336183371833818339183401834118342183431834418345183461834718348183491835018351183521835318354183551835618357183581835918360183611836218363183641836518366183671836818369183701837118372183731837418375183761837718378183791838018381183821838318384183851838618387183881838918390183911839218393183941839518396183971839818399184001840118402184031840418405184061840718408184091841018411184121841318414184151841618417184181841918420184211842218423184241842518426184271842818429184301843118432184331843418435184361843718438184391844018441184421844318444184451844618447184481844918450184511845218453184541845518456184571845818459184601846118462184631846418465184661846718468184691847018471184721847318474184751847618477184781847918480184811848218483184841848518486184871848818489184901849118492184931849418495184961849718498184991850018501185021850318504185051850618507185081850918510185111851218513185141851518516185171851818519185201852118522185231852418525185261852718528185291853018531185321853318534185351853618537185381853918540185411854218543185441854518546185471854818549185501855118552185531855418555185561855718558185591856018561185621856318564185651856618567185681856918570185711857218573185741857518576185771857818579185801858118582185831858418585185861858718588185891859018591185921859318594185951859618597185981859918600186011860218603186041860518606186071860818609186101861118612186131861418615186161861718618186191862018621186221862318624186251862618627186281862918630186311863218633186341863518636186371863818639186401864118642186431864418645186461864718648186491865018651186521865318654186551865618657186581865918660186611866218663186641866518666186671866818669186701867118672186731867418675186761867718678186791868018681186821868318684186851868618687186881868918690186911869218693186941869518696186971869818699187001870118702187031870418705187061870718708187091871018711187121871318714187151871618717187181871918720187211872218723187241872518726187271872818729187301873118732187331873418735187361873718738187391874018741187421874318744187451874618747187481874918750187511875218753187541875518756187571875818759187601876118762187631876418765187661876718768187691877018771187721877318774187751877618777187781877918780187811878218783187841878518786187871878818789187901879118792187931879418795187961879718798187991880018801188021880318804188051880618807188081880918810188111881218813188141881518816188171881818819188201882118822188231882418825188261882718828188291883018831188321883318834188351883618837188381883918840188411884218843188441884518846188471884818849188501885118852188531885418855188561885718858188591886018861188621886318864188651886618867188681886918870188711887218873188741887518876188771887818879188801888118882188831888418885188861888718888188891889018891188921889318894188951889618897188981889918900189011890218903189041890518906189071890818909189101891118912189131891418915189161891718918189191892018921189221892318924189251892618927189281892918930189311893218933189341893518936189371893818939189401894118942189431894418945189461894718948189491895018951189521895318954189551895618957189581895918960189611896218963189641896518966189671896818969189701897118972189731897418975189761897718978189791898018981189821898318984189851898618987189881898918990189911899218993189941899518996189971899818999190001900119002190031900419005190061900719008190091901019011190121901319014190151901619017190181901919020190211902219023190241902519026190271902819029190301903119032190331903419035190361903719038190391904019041190421904319044190451904619047190481904919050190511905219053190541905519056190571905819059190601906119062190631906419065190661906719068190691907019071190721907319074190751907619077190781907919080190811908219083190841908519086190871908819089190901909119092190931909419095190961909719098190991910019101191021910319104191051910619107191081910919110191111911219113191141911519116191171911819119191201912119122191231912419125191261912719128191291913019131191321913319134191351913619137191381913919140191411914219143191441914519146191471914819149191501915119152191531915419155191561915719158191591916019161191621916319164191651916619167191681916919170191711917219173191741917519176191771917819179191801918119182191831918419185191861918719188191891919019191191921919319194191951919619197191981919919200192011920219203192041920519206192071920819209192101921119212192131921419215192161921719218192191922019221192221922319224192251922619227192281922919230192311923219233192341923519236192371923819239192401924119242192431924419245192461924719248192491925019251192521925319254192551925619257192581925919260192611926219263192641926519266192671926819269192701927119272192731927419275192761927719278192791928019281192821928319284192851928619287192881928919290192911929219293192941929519296192971929819299193001930119302193031930419305193061930719308193091931019311193121931319314193151931619317193181931919320193211932219323193241932519326193271932819329193301933119332193331933419335193361933719338193391934019341193421934319344193451934619347193481934919350193511935219353193541935519356193571935819359193601936119362193631936419365193661936719368193691937019371193721937319374193751937619377193781937919380193811938219383193841938519386193871938819389193901939119392193931939419395193961939719398193991940019401194021940319404194051940619407194081940919410194111941219413194141941519416194171941819419194201942119422194231942419425194261942719428194291943019431194321943319434194351943619437194381943919440194411944219443194441944519446194471944819449194501945119452194531945419455194561945719458194591946019461194621946319464194651946619467194681946919470194711947219473194741947519476194771947819479194801948119482194831948419485194861948719488194891949019491194921949319494194951949619497194981949919500195011950219503195041950519506195071950819509195101951119512195131951419515195161951719518195191952019521195221952319524195251952619527195281952919530195311953219533195341953519536195371953819539195401954119542195431954419545195461954719548195491955019551195521955319554195551955619557195581955919560195611956219563195641956519566195671956819569195701957119572195731957419575195761957719578195791958019581195821958319584195851958619587195881958919590195911959219593195941959519596195971959819599196001960119602196031960419605196061960719608196091961019611196121961319614196151961619617196181961919620196211962219623196241962519626196271962819629196301963119632196331963419635196361963719638196391964019641196421964319644196451964619647196481964919650196511965219653196541965519656196571965819659196601966119662196631966419665196661966719668196691967019671196721967319674196751967619677196781967919680196811968219683196841968519686196871968819689196901969119692196931969419695196961969719698196991970019701197021970319704197051970619707197081970919710197111971219713197141971519716197171971819719197201972119722197231972419725197261972719728197291973019731197321973319734197351973619737197381973919740197411974219743197441974519746197471974819749197501975119752197531975419755197561975719758197591976019761197621976319764197651976619767197681976919770197711977219773197741977519776197771977819779197801978119782197831978419785197861978719788197891979019791197921979319794197951979619797197981979919800198011980219803198041980519806198071980819809198101981119812198131981419815198161981719818198191982019821198221982319824198251982619827198281982919830198311983219833198341983519836198371983819839198401984119842198431984419845198461984719848198491985019851198521985319854198551985619857198581985919860198611986219863198641986519866198671986819869198701987119872198731987419875198761987719878198791988019881198821988319884198851988619887198881988919890198911989219893198941989519896198971989819899199001990119902199031990419905199061990719908199091991019911199121991319914199151991619917199181991919920199211992219923199241992519926199271992819929199301993119932199331993419935199361993719938199391994019941199421994319944199451994619947199481994919950199511995219953199541995519956199571995819959199601996119962199631996419965199661996719968199691997019971199721997319974199751997619977199781997919980199811998219983199841998519986199871998819989199901999119992199931999419995199961999719998199992000020001200022000320004200052000620007200082000920010200112001220013200142001520016200172001820019200202002120022200232002420025200262002720028200292003020031200322003320034200352003620037200382003920040200412004220043200442004520046200472004820049200502005120052200532005420055200562005720058200592006020061200622006320064200652006620067200682006920070200712007220073200742007520076200772007820079200802008120082200832008420085200862008720088200892009020091200922009320094200952009620097200982009920100201012010220103201042010520106201072010820109201102011120112201132011420115201162011720118201192012020121201222012320124201252012620127201282012920130201312013220133201342013520136201372013820139201402014120142201432014420145201462014720148201492015020151201522015320154201552015620157201582015920160201612016220163201642016520166201672016820169201702017120172201732017420175201762017720178201792018020181201822018320184201852018620187201882018920190201912019220193201942019520196201972019820199202002020120202202032020420205202062020720208202092021020211202122021320214202152021620217202182021920220202212022220223202242022520226202272022820229202302023120232202332023420235202362023720238202392024020241202422024320244202452024620247202482024920250202512025220253202542025520256202572025820259202602026120262202632026420265202662026720268202692027020271202722027320274202752027620277202782027920280202812028220283202842028520286202872028820289202902029120292202932029420295202962029720298202992030020301203022030320304203052030620307203082030920310203112031220313203142031520316203172031820319203202032120322203232032420325203262032720328203292033020331203322033320334203352033620337203382033920340203412034220343203442034520346203472034820349203502035120352203532035420355203562035720358203592036020361203622036320364203652036620367203682036920370203712037220373203742037520376203772037820379203802038120382203832038420385203862038720388203892039020391203922039320394203952039620397203982039920400204012040220403204042040520406204072040820409204102041120412204132041420415204162041720418204192042020421204222042320424204252042620427204282042920430204312043220433204342043520436204372043820439204402044120442204432044420445204462044720448204492045020451204522045320454204552045620457204582045920460204612046220463204642046520466204672046820469204702047120472204732047420475204762047720478204792048020481204822048320484204852048620487204882048920490204912049220493204942049520496204972049820499205002050120502205032050420505205062050720508205092051020511205122051320514205152051620517205182051920520205212052220523205242052520526205272052820529205302053120532205332053420535205362053720538205392054020541205422054320544205452054620547205482054920550205512055220553205542055520556205572055820559205602056120562205632056420565205662056720568205692057020571205722057320574205752057620577205782057920580205812058220583205842058520586205872058820589205902059120592205932059420595205962059720598205992060020601206022060320604206052060620607206082060920610206112061220613206142061520616206172061820619206202062120622206232062420625206262062720628206292063020631206322063320634206352063620637206382063920640206412064220643206442064520646206472064820649206502065120652206532065420655206562065720658206592066020661206622066320664206652066620667206682066920670206712067220673206742067520676206772067820679206802068120682206832068420685206862068720688206892069020691206922069320694206952069620697206982069920700207012070220703207042070520706207072070820709207102071120712207132071420715207162071720718207192072020721207222072320724207252072620727207282072920730207312073220733207342073520736207372073820739207402074120742207432074420745207462074720748207492075020751207522075320754207552075620757207582075920760207612076220763207642076520766207672076820769207702077120772207732077420775207762077720778207792078020781207822078320784207852078620787207882078920790207912079220793207942079520796207972079820799208002080120802208032080420805208062080720808208092081020811208122081320814208152081620817208182081920820208212082220823208242082520826208272082820829208302083120832208332083420835208362083720838208392084020841208422084320844208452084620847208482084920850208512085220853208542085520856208572085820859208602086120862208632086420865208662086720868208692087020871208722087320874208752087620877208782087920880208812088220883208842088520886208872088820889208902089120892208932089420895208962089720898208992090020901209022090320904209052090620907209082090920910209112091220913209142091520916209172091820919209202092120922209232092420925209262092720928209292093020931209322093320934209352093620937209382093920940209412094220943209442094520946209472094820949209502095120952209532095420955209562095720958209592096020961209622096320964209652096620967209682096920970209712097220973209742097520976209772097820979209802098120982209832098420985209862098720988209892099020991209922099320994209952099620997209982099921000210012100221003210042100521006210072100821009210102101121012210132101421015210162101721018210192102021021210222102321024210252102621027210282102921030210312103221033210342103521036210372103821039210402104121042210432104421045210462104721048210492105021051210522105321054210552105621057210582105921060210612106221063210642106521066210672106821069210702107121072210732107421075210762107721078210792108021081210822108321084210852108621087210882108921090210912109221093210942109521096210972109821099211002110121102211032110421105211062110721108211092111021111211122111321114211152111621117211182111921120211212112221123211242112521126211272112821129211302113121132211332113421135211362113721138211392114021141211422114321144211452114621147211482114921150211512115221153211542115521156211572115821159211602116121162211632116421165211662116721168211692117021171211722117321174211752117621177211782117921180211812118221183211842118521186211872118821189211902119121192211932119421195211962119721198211992120021201212022120321204212052120621207212082120921210212112121221213212142121521216212172121821219212202122121222212232122421225212262122721228212292123021231212322123321234212352123621237212382123921240212412124221243212442124521246212472124821249212502125121252212532125421255212562125721258212592126021261212622126321264212652126621267212682126921270212712127221273212742127521276212772127821279212802128121282212832128421285212862128721288212892129021291212922129321294212952129621297212982129921300213012130221303213042130521306213072130821309213102131121312213132131421315213162131721318213192132021321213222132321324213252132621327213282132921330213312133221333213342133521336213372133821339213402134121342213432134421345213462134721348213492135021351213522135321354213552135621357213582135921360213612136221363213642136521366213672136821369213702137121372213732137421375213762137721378213792138021381213822138321384213852138621387213882138921390213912139221393213942139521396213972139821399214002140121402214032140421405214062140721408214092141021411214122141321414214152141621417214182141921420214212142221423214242142521426214272142821429214302143121432214332143421435214362143721438214392144021441214422144321444214452144621447214482144921450214512145221453214542145521456214572145821459214602146121462214632146421465214662146721468214692147021471214722147321474214752147621477214782147921480214812148221483214842148521486214872148821489214902149121492214932149421495214962149721498214992150021501215022150321504215052150621507215082150921510215112151221513215142151521516215172151821519215202152121522215232152421525215262152721528215292153021531215322153321534215352153621537215382153921540215412154221543215442154521546215472154821549215502155121552215532155421555215562155721558215592156021561215622156321564215652156621567215682156921570215712157221573215742157521576215772157821579215802158121582215832158421585215862158721588215892159021591215922159321594215952159621597215982159921600216012160221603216042160521606216072160821609216102161121612216132161421615216162161721618216192162021621216222162321624216252162621627216282162921630216312163221633216342163521636216372163821639216402164121642216432164421645216462164721648216492165021651216522165321654216552165621657216582165921660216612166221663216642166521666216672166821669216702167121672216732167421675216762167721678216792168021681216822168321684216852168621687216882168921690216912169221693216942169521696216972169821699217002170121702217032170421705217062170721708217092171021711217122171321714217152171621717217182171921720217212172221723217242172521726217272172821729217302173121732217332173421735217362173721738217392174021741217422174321744217452174621747217482174921750217512175221753217542175521756217572175821759217602176121762217632176421765217662176721768217692177021771217722177321774217752177621777217782177921780217812178221783217842178521786217872178821789217902179121792217932179421795217962179721798217992180021801218022180321804218052180621807218082180921810218112181221813218142181521816218172181821819218202182121822218232182421825218262182721828218292183021831218322183321834218352183621837218382183921840218412184221843218442184521846218472184821849218502185121852218532185421855218562185721858218592186021861218622186321864218652186621867218682186921870218712187221873218742187521876218772187821879218802188121882218832188421885218862188721888218892189021891218922189321894218952189621897218982189921900219012190221903219042190521906219072190821909219102191121912219132191421915219162191721918219192192021921219222192321924219252192621927219282192921930219312193221933219342193521936219372193821939219402194121942219432194421945219462194721948219492195021951219522195321954219552195621957219582195921960219612196221963219642196521966219672196821969219702197121972219732197421975219762197721978219792198021981219822198321984219852198621987219882198921990219912199221993219942199521996219972199821999220002200122002220032200422005220062200722008220092201022011220122201322014220152201622017220182201922020220212202222023220242202522026220272202822029220302203122032220332203422035220362203722038220392204022041220422204322044220452204622047220482204922050220512205222053220542205522056220572205822059220602206122062220632206422065220662206722068220692207022071220722207322074220752207622077220782207922080220812208222083220842208522086220872208822089220902209122092220932209422095220962209722098220992210022101221022210322104221052210622107221082210922110221112211222113221142211522116221172211822119221202212122122221232212422125221262212722128221292213022131221322213322134221352213622137221382213922140221412214222143221442214522146221472214822149221502215122152221532215422155221562215722158221592216022161221622216322164221652216622167221682216922170221712217222173221742217522176221772217822179221802218122182221832218422185221862218722188221892219022191221922219322194221952219622197221982219922200222012220222203222042220522206222072220822209222102221122212222132221422215222162221722218222192222022221222222222322224222252222622227222282222922230222312223222233222342223522236222372223822239222402224122242222432224422245222462224722248222492225022251222522225322254222552225622257222582225922260222612226222263222642226522266222672226822269222702227122272222732227422275222762227722278222792228022281222822228322284222852228622287222882228922290222912229222293222942229522296222972229822299223002230122302223032230422305223062230722308223092231022311223122231322314223152231622317223182231922320223212232222323223242232522326223272232822329223302233122332223332233422335223362233722338223392234022341223422234322344223452234622347223482234922350223512235222353223542235522356223572235822359223602236122362223632236422365223662236722368223692237022371223722237322374223752237622377223782237922380223812238222383223842238522386223872238822389223902239122392223932239422395223962239722398223992240022401224022240322404224052240622407224082240922410224112241222413224142241522416224172241822419224202242122422224232242422425224262242722428224292243022431224322243322434224352243622437224382243922440224412244222443224442244522446224472244822449224502245122452224532245422455224562245722458224592246022461224622246322464224652246622467224682246922470224712247222473224742247522476224772247822479224802248122482224832248422485224862248722488224892249022491224922249322494224952249622497224982249922500225012250222503225042250522506225072250822509225102251122512225132251422515225162251722518225192252022521225222252322524225252252622527225282252922530225312253222533225342253522536225372253822539225402254122542225432254422545225462254722548225492255022551225522255322554225552255622557225582255922560225612256222563225642256522566225672256822569225702257122572225732257422575225762257722578225792258022581225822258322584225852258622587225882258922590225912259222593225942259522596225972259822599226002260122602226032260422605226062260722608226092261022611226122261322614226152261622617226182261922620226212262222623226242262522626226272262822629226302263122632226332263422635226362263722638226392264022641226422264322644226452264622647226482264922650226512265222653226542265522656226572265822659226602266122662226632266422665226662266722668226692267022671226722267322674226752267622677226782267922680226812268222683226842268522686226872268822689226902269122692226932269422695226962269722698226992270022701227022270322704227052270622707227082270922710227112271222713227142271522716227172271822719227202272122722227232272422725227262272722728227292273022731227322273322734227352273622737227382273922740227412274222743227442274522746227472274822749227502275122752227532275422755227562275722758227592276022761227622276322764227652276622767227682276922770227712277222773227742277522776227772277822779227802278122782227832278422785227862278722788227892279022791227922279322794227952279622797227982279922800228012280222803228042280522806228072280822809228102281122812228132281422815228162281722818228192282022821228222282322824228252282622827228282282922830228312283222833228342283522836228372283822839228402284122842228432284422845228462284722848228492285022851228522285322854228552285622857228582285922860228612286222863228642286522866228672286822869228702287122872228732287422875228762287722878228792288022881228822288322884228852288622887228882288922890228912289222893228942289522896228972289822899229002290122902229032290422905229062290722908229092291022911229122291322914229152291622917229182291922920229212292222923229242292522926229272292822929229302293122932229332293422935229362293722938229392294022941229422294322944229452294622947229482294922950229512295222953229542295522956229572295822959229602296122962229632296422965229662296722968229692297022971229722297322974229752297622977229782297922980229812298222983229842298522986229872298822989229902299122992229932299422995229962299722998229992300023001230022300323004230052300623007230082300923010230112301223013230142301523016230172301823019230202302123022230232302423025230262302723028230292303023031230322303323034230352303623037230382303923040230412304223043230442304523046230472304823049230502305123052230532305423055230562305723058230592306023061230622306323064230652306623067230682306923070230712307223073230742307523076230772307823079230802308123082230832308423085230862308723088230892309023091230922309323094230952309623097230982309923100231012310223103231042310523106231072310823109231102311123112231132311423115231162311723118231192312023121231222312323124231252312623127231282312923130231312313223133231342313523136231372313823139231402314123142231432314423145231462314723148231492315023151231522315323154231552315623157231582315923160231612316223163231642316523166231672316823169231702317123172231732317423175231762317723178231792318023181231822318323184231852318623187231882318923190231912319223193231942319523196231972319823199232002320123202232032320423205232062320723208232092321023211232122321323214232152321623217232182321923220232212322223223232242322523226232272322823229232302323123232232332323423235232362323723238232392324023241232422324323244232452324623247232482324923250232512325223253232542325523256232572325823259232602326123262232632326423265232662326723268232692327023271232722327323274232752327623277232782327923280232812328223283232842328523286232872328823289232902329123292232932329423295232962329723298232992330023301233022330323304233052330623307233082330923310233112331223313233142331523316233172331823319233202332123322233232332423325233262332723328233292333023331233322333323334233352333623337233382333923340233412334223343233442334523346233472334823349233502335123352233532335423355233562335723358233592336023361233622336323364233652336623367233682336923370233712337223373233742337523376233772337823379233802338123382233832338423385233862338723388233892339023391233922339323394233952339623397233982339923400234012340223403234042340523406234072340823409234102341123412234132341423415234162341723418234192342023421234222342323424234252342623427234282342923430234312343223433234342343523436234372343823439234402344123442234432344423445234462344723448234492345023451234522345323454234552345623457234582345923460234612346223463234642346523466234672346823469234702347123472234732347423475234762347723478234792348023481234822348323484234852348623487234882348923490234912349223493234942349523496234972349823499235002350123502235032350423505235062350723508235092351023511235122351323514235152351623517235182351923520235212352223523235242352523526235272352823529235302353123532235332353423535235362353723538235392354023541235422354323544235452354623547235482354923550235512355223553235542355523556235572355823559235602356123562235632356423565235662356723568235692357023571235722357323574235752357623577235782357923580235812358223583235842358523586235872358823589235902359123592235932359423595235962359723598235992360023601236022360323604236052360623607236082360923610236112361223613236142361523616236172361823619236202362123622236232362423625236262362723628236292363023631236322363323634236352363623637236382363923640236412364223643236442364523646236472364823649236502365123652236532365423655236562365723658236592366023661236622366323664236652366623667236682366923670236712367223673236742367523676236772367823679236802368123682236832368423685236862368723688236892369023691236922369323694236952369623697236982369923700237012370223703237042370523706237072370823709237102371123712237132371423715237162371723718237192372023721237222372323724237252372623727237282372923730237312373223733237342373523736237372373823739237402374123742237432374423745237462374723748237492375023751237522375323754237552375623757237582375923760237612376223763237642376523766237672376823769237702377123772237732377423775237762377723778237792378023781237822378323784237852378623787237882378923790237912379223793237942379523796237972379823799238002380123802238032380423805238062380723808238092381023811238122381323814238152381623817238182381923820238212382223823238242382523826238272382823829238302383123832238332383423835238362383723838238392384023841238422384323844238452384623847238482384923850238512385223853238542385523856238572385823859238602386123862238632386423865238662386723868238692387023871238722387323874238752387623877238782387923880238812388223883238842388523886238872388823889238902389123892238932389423895238962389723898238992390023901239022390323904239052390623907239082390923910239112391223913239142391523916239172391823919239202392123922239232392423925239262392723928239292393023931239322393323934239352393623937239382393923940239412394223943239442394523946239472394823949239502395123952239532395423955239562395723958239592396023961239622396323964239652396623967239682396923970239712397223973239742397523976239772397823979239802398123982239832398423985239862398723988239892399023991239922399323994239952399623997239982399924000240012400224003240042400524006240072400824009240102401124012240132401424015240162401724018240192402024021240222402324024240252402624027240282402924030240312403224033240342403524036240372403824039240402404124042240432404424045240462404724048240492405024051240522405324054240552405624057240582405924060240612406224063240642406524066240672406824069240702407124072240732407424075240762407724078240792408024081240822408324084240852408624087240882408924090240912409224093240942409524096240972409824099241002410124102241032410424105241062410724108241092411024111241122411324114241152411624117241182411924120241212412224123241242412524126241272412824129241302413124132241332413424135241362413724138241392414024141241422414324144241452414624147241482414924150241512415224153241542415524156241572415824159241602416124162241632416424165241662416724168241692417024171241722417324174241752417624177241782417924180241812418224183241842418524186241872418824189241902419124192241932419424195241962419724198241992420024201242022420324204242052420624207242082420924210242112421224213242142421524216242172421824219242202422124222242232422424225242262422724228242292423024231242322423324234242352423624237242382423924240242412424224243242442424524246242472424824249242502425124252242532425424255242562425724258242592426024261242622426324264242652426624267242682426924270242712427224273242742427524276242772427824279242802428124282242832428424285242862428724288242892429024291242922429324294242952429624297242982429924300243012430224303243042430524306243072430824309243102431124312243132431424315243162431724318243192432024321243222432324324243252432624327243282432924330243312433224333243342433524336243372433824339243402434124342243432434424345243462434724348243492435024351243522435324354243552435624357243582435924360243612436224363243642436524366243672436824369243702437124372243732437424375243762437724378243792438024381243822438324384243852438624387243882438924390243912439224393243942439524396243972439824399244002440124402244032440424405244062440724408244092441024411244122441324414244152441624417244182441924420244212442224423244242442524426244272442824429244302443124432244332443424435244362443724438244392444024441244422444324444244452444624447244482444924450244512445224453244542445524456244572445824459244602446124462244632446424465244662446724468244692447024471244722447324474244752447624477244782447924480244812448224483244842448524486244872448824489244902449124492244932449424495244962449724498244992450024501245022450324504245052450624507245082450924510245112451224513245142451524516245172451824519245202452124522245232452424525245262452724528245292453024531245322453324534245352453624537245382453924540245412454224543245442454524546245472454824549245502455124552245532455424555245562455724558245592456024561245622456324564245652456624567245682456924570245712457224573245742457524576245772457824579245802458124582245832458424585245862458724588245892459024591245922459324594245952459624597245982459924600246012460224603246042460524606246072460824609246102461124612246132461424615246162461724618246192462024621246222462324624246252462624627246282462924630246312463224633246342463524636246372463824639246402464124642246432464424645246462464724648246492465024651246522465324654246552465624657246582465924660246612466224663246642466524666246672466824669246702467124672246732467424675246762467724678246792468024681246822468324684246852468624687246882468924690246912469224693246942469524696246972469824699247002470124702247032470424705247062470724708247092471024711247122471324714247152471624717247182471924720247212472224723247242472524726247272472824729247302473124732247332473424735247362473724738247392474024741247422474324744247452474624747247482474924750247512475224753247542475524756247572475824759247602476124762247632476424765247662476724768247692477024771247722477324774247752477624777247782477924780247812478224783247842478524786247872478824789247902479124792247932479424795247962479724798247992480024801248022480324804248052480624807248082480924810248112481224813248142481524816248172481824819248202482124822248232482424825248262482724828248292483024831248322483324834248352483624837248382483924840248412484224843248442484524846248472484824849248502485124852248532485424855248562485724858248592486024861248622486324864248652486624867248682486924870248712487224873248742487524876248772487824879248802488124882248832488424885248862488724888248892489024891248922489324894248952489624897248982489924900249012490224903249042490524906249072490824909249102491124912249132491424915249162491724918249192492024921249222492324924249252492624927249282492924930249312493224933249342493524936249372493824939249402494124942249432494424945249462494724948249492495024951249522495324954249552495624957249582495924960249612496224963249642496524966249672496824969249702497124972249732497424975249762497724978249792498024981249822498324984249852498624987249882498924990249912499224993249942499524996249972499824999250002500125002250032500425005250062500725008250092501025011250122501325014250152501625017250182501925020250212502225023250242502525026250272502825029250302503125032250332503425035250362503725038250392504025041250422504325044250452504625047250482504925050250512505225053250542505525056250572505825059250602506125062250632506425065250662506725068250692507025071250722507325074250752507625077250782507925080250812508225083250842508525086250872508825089250902509125092250932509425095250962509725098250992510025101251022510325104251052510625107251082510925110251112511225113251142511525116251172511825119251202512125122251232512425125251262512725128251292513025131251322513325134251352513625137251382513925140251412514225143251442514525146251472514825149251502515125152251532515425155251562515725158251592516025161251622516325164251652516625167251682516925170251712517225173251742517525176251772517825179251802518125182251832518425185251862518725188251892519025191251922519325194251952519625197251982519925200252012520225203252042520525206252072520825209252102521125212252132521425215252162521725218252192522025221252222522325224252252522625227252282522925230252312523225233252342523525236252372523825239252402524125242252432524425245252462524725248252492525025251252522525325254252552525625257252582525925260252612526225263252642526525266252672526825269252702527125272252732527425275252762527725278252792528025281252822528325284252852528625287252882528925290252912529225293252942529525296252972529825299253002530125302253032530425305253062530725308253092531025311253122531325314253152531625317253182531925320253212532225323253242532525326253272532825329253302533125332253332533425335253362533725338253392534025341253422534325344253452534625347253482534925350253512535225353253542535525356253572535825359253602536125362253632536425365253662536725368253692537025371253722537325374253752537625377253782537925380253812538225383253842538525386253872538825389253902539125392253932539425395253962539725398253992540025401254022540325404254052540625407254082540925410254112541225413254142541525416254172541825419254202542125422254232542425425254262542725428254292543025431254322543325434254352543625437254382543925440254412544225443254442544525446254472544825449254502545125452254532545425455254562545725458254592546025461254622546325464254652546625467254682546925470254712547225473254742547525476254772547825479254802548125482254832548425485254862548725488254892549025491254922549325494254952549625497254982549925500255012550225503255042550525506255072550825509255102551125512255132551425515255162551725518255192552025521255222552325524255252552625527255282552925530255312553225533255342553525536255372553825539255402554125542255432554425545255462554725548255492555025551255522555325554255552555625557255582555925560255612556225563255642556525566255672556825569255702557125572255732557425575255762557725578255792558025581255822558325584255852558625587255882558925590255912559225593255942559525596255972559825599256002560125602256032560425605256062560725608256092561025611256122561325614256152561625617256182561925620256212562225623256242562525626256272562825629256302563125632256332563425635256362563725638256392564025641256422564325644256452564625647256482564925650256512565225653256542565525656256572565825659256602566125662256632566425665256662566725668256692567025671256722567325674256752567625677256782567925680256812568225683256842568525686256872568825689256902569125692256932569425695256962569725698256992570025701257022570325704257052570625707257082570925710257112571225713257142571525716257172571825719257202572125722257232572425725257262572725728257292573025731257322573325734257352573625737257382573925740257412574225743257442574525746257472574825749257502575125752257532575425755257562575725758257592576025761257622576325764257652576625767257682576925770257712577225773257742577525776257772577825779257802578125782257832578425785257862578725788257892579025791257922579325794257952579625797257982579925800258012580225803258042580525806258072580825809258102581125812258132581425815258162581725818258192582025821258222582325824258252582625827258282582925830258312583225833258342583525836258372583825839258402584125842258432584425845258462584725848258492585025851258522585325854258552585625857258582585925860258612586225863258642586525866258672586825869258702587125872258732587425875258762587725878258792588025881258822588325884258852588625887258882588925890258912589225893258942589525896258972589825899259002590125902259032590425905259062590725908259092591025911259122591325914259152591625917259182591925920259212592225923259242592525926259272592825929259302593125932259332593425935259362593725938259392594025941259422594325944259452594625947259482594925950259512595225953259542595525956259572595825959259602596125962259632596425965259662596725968259692597025971259722597325974259752597625977259782597925980259812598225983259842598525986259872598825989259902599125992259932599425995259962599725998259992600026001260022600326004260052600626007260082600926010260112601226013260142601526016260172601826019260202602126022260232602426025260262602726028260292603026031260322603326034260352603626037260382603926040260412604226043260442604526046260472604826049260502605126052260532605426055260562605726058260592606026061260622606326064260652606626067260682606926070260712607226073260742607526076260772607826079260802608126082260832608426085260862608726088260892609026091260922609326094260952609626097260982609926100261012610226103261042610526106261072610826109261102611126112261132611426115261162611726118261192612026121261222612326124261252612626127261282612926130261312613226133261342613526136261372613826139261402614126142261432614426145261462614726148261492615026151261522615326154261552615626157261582615926160261612616226163261642616526166261672616826169261702617126172261732617426175261762617726178261792618026181261822618326184261852618626187261882618926190261912619226193261942619526196261972619826199262002620126202262032620426205262062620726208262092621026211262122621326214262152621626217262182621926220262212622226223262242622526226262272622826229262302623126232
  1. diff -Nur linux-4.6.6.orig/arch/arm/include/asm/switch_to.h linux-4.6.6/arch/arm/include/asm/switch_to.h
  2. --- linux-4.6.6.orig/arch/arm/include/asm/switch_to.h 2016-08-10 12:54:57.000000000 +0200
  3. +++ linux-4.6.6/arch/arm/include/asm/switch_to.h 2016-08-22 21:27:46.269819745 +0200
  4. @@ -3,6 +3,13 @@
  5. #include <linux/thread_info.h>
  6. +#if defined CONFIG_PREEMPT_RT_FULL && defined CONFIG_HIGHMEM
  7. +void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p);
  8. +#else
  9. +static inline void
  10. +switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p) { }
  11. +#endif
  12. +
  13. /*
  14. * For v7 SMP cores running a preemptible kernel we may be pre-empted
  15. * during a TLB maintenance operation, so execute an inner-shareable dsb
  16. @@ -25,6 +32,7 @@
  17. #define switch_to(prev,next,last) \
  18. do { \
  19. __complete_pending_tlbi(); \
  20. + switch_kmaps(prev, next); \
  21. last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
  22. } while (0)
  23. diff -Nur linux-4.6.6.orig/arch/arm/include/asm/thread_info.h linux-4.6.6/arch/arm/include/asm/thread_info.h
  24. --- linux-4.6.6.orig/arch/arm/include/asm/thread_info.h 2016-08-10 12:54:57.000000000 +0200
  25. +++ linux-4.6.6/arch/arm/include/asm/thread_info.h 2016-08-22 21:27:46.269819745 +0200
  26. @@ -49,6 +49,7 @@
  27. struct thread_info {
  28. unsigned long flags; /* low level flags */
  29. int preempt_count; /* 0 => preemptable, <0 => bug */
  30. + int preempt_lazy_count; /* 0 => preemptable, <0 => bug */
  31. mm_segment_t addr_limit; /* address limit */
  32. struct task_struct *task; /* main task structure */
  33. __u32 cpu; /* cpu */
  34. @@ -142,7 +143,8 @@
  35. #define TIF_SYSCALL_TRACE 4 /* syscall trace active */
  36. #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */
  37. #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */
  38. -#define TIF_SECCOMP 7 /* seccomp syscall filtering active */
  39. +#define TIF_SECCOMP 8 /* seccomp syscall filtering active */
  40. +#define TIF_NEED_RESCHED_LAZY 7
  41. #define TIF_NOHZ 12 /* in adaptive nohz mode */
  42. #define TIF_USING_IWMMXT 17
  43. @@ -152,6 +154,7 @@
  44. #define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
  45. #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
  46. #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
  47. +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
  48. #define _TIF_UPROBE (1 << TIF_UPROBE)
  49. #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
  50. #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
  51. @@ -167,7 +170,8 @@
  52. * Change these and you break ASM code in entry-common.S
  53. */
  54. #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
  55. - _TIF_NOTIFY_RESUME | _TIF_UPROBE)
  56. + _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
  57. + _TIF_NEED_RESCHED_LAZY)
  58. #endif /* __KERNEL__ */
  59. #endif /* __ASM_ARM_THREAD_INFO_H */
  60. diff -Nur linux-4.6.6.orig/arch/arm/Kconfig linux-4.6.6/arch/arm/Kconfig
  61. --- linux-4.6.6.orig/arch/arm/Kconfig 2016-08-10 12:54:57.000000000 +0200
  62. +++ linux-4.6.6/arch/arm/Kconfig 2016-08-22 21:27:46.261819236 +0200
  63. @@ -35,7 +35,7 @@
  64. select HARDIRQS_SW_RESEND
  65. select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT)
  66. select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6
  67. - select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU
  68. + select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT_BASE
  69. select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
  70. select HAVE_ARCH_MMAP_RND_BITS if MMU
  71. select HAVE_ARCH_SECCOMP_FILTER if (AEABI && !OABI_COMPAT)
  72. @@ -71,6 +71,7 @@
  73. select HAVE_PERF_EVENTS
  74. select HAVE_PERF_REGS
  75. select HAVE_PERF_USER_STACK_DUMP
  76. + select HAVE_PREEMPT_LAZY
  77. select HAVE_RCU_TABLE_FREE if (SMP && ARM_LPAE)
  78. select HAVE_REGS_AND_STACK_ACCESS_API
  79. select HAVE_SYSCALL_TRACEPOINTS
  80. diff -Nur linux-4.6.6.orig/arch/arm/kernel/asm-offsets.c linux-4.6.6/arch/arm/kernel/asm-offsets.c
  81. --- linux-4.6.6.orig/arch/arm/kernel/asm-offsets.c 2016-08-10 12:54:57.000000000 +0200
  82. +++ linux-4.6.6/arch/arm/kernel/asm-offsets.c 2016-08-22 21:27:46.293821284 +0200
  83. @@ -65,6 +65,7 @@
  84. BLANK();
  85. DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
  86. DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
  87. + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count));
  88. DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit));
  89. DEFINE(TI_TASK, offsetof(struct thread_info, task));
  90. DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
  91. diff -Nur linux-4.6.6.orig/arch/arm/kernel/entry-armv.S linux-4.6.6/arch/arm/kernel/entry-armv.S
  92. --- linux-4.6.6.orig/arch/arm/kernel/entry-armv.S 2016-08-10 12:54:57.000000000 +0200
  93. +++ linux-4.6.6/arch/arm/kernel/entry-armv.S 2016-08-22 21:27:46.305822058 +0200
  94. @@ -215,11 +215,18 @@
  95. #ifdef CONFIG_PREEMPT
  96. get_thread_info tsk
  97. ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
  98. - ldr r0, [tsk, #TI_FLAGS] @ get flags
  99. teq r8, #0 @ if preempt count != 0
  100. + bne 1f @ return from exeption
  101. + ldr r0, [tsk, #TI_FLAGS] @ get flags
  102. + tst r0, #_TIF_NEED_RESCHED @ if NEED_RESCHED is set
  103. + blne svc_preempt @ preempt!
  104. +
  105. + ldr r8, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count
  106. + teq r8, #0 @ if preempt lazy count != 0
  107. movne r0, #0 @ force flags to 0
  108. - tst r0, #_TIF_NEED_RESCHED
  109. + tst r0, #_TIF_NEED_RESCHED_LAZY
  110. blne svc_preempt
  111. +1:
  112. #endif
  113. svc_exit r5, irq = 1 @ return from exception
  114. @@ -234,8 +241,14 @@
  115. 1: bl preempt_schedule_irq @ irq en/disable is done inside
  116. ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
  117. tst r0, #_TIF_NEED_RESCHED
  118. + bne 1b
  119. + tst r0, #_TIF_NEED_RESCHED_LAZY
  120. reteq r8 @ go again
  121. - b 1b
  122. + ldr r0, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count
  123. + teq r0, #0 @ if preempt lazy count != 0
  124. + beq 1b
  125. + ret r8 @ go again
  126. +
  127. #endif
  128. __und_fault:
  129. diff -Nur linux-4.6.6.orig/arch/arm/kernel/entry-common.S linux-4.6.6/arch/arm/kernel/entry-common.S
  130. --- linux-4.6.6.orig/arch/arm/kernel/entry-common.S 2016-08-10 12:54:57.000000000 +0200
  131. +++ linux-4.6.6/arch/arm/kernel/entry-common.S 2016-08-22 21:27:46.305822058 +0200
  132. @@ -36,7 +36,9 @@
  133. UNWIND(.cantunwind )
  134. disable_irq_notrace @ disable interrupts
  135. ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
  136. - tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
  137. + tst r1, #((_TIF_SYSCALL_WORK | _TIF_WORK_MASK) & ~_TIF_SECCOMP)
  138. + bne fast_work_pending
  139. + tst r1, #_TIF_SECCOMP
  140. bne fast_work_pending
  141. /* perform architecture specific actions before user return */
  142. @@ -62,8 +64,11 @@
  143. str r0, [sp, #S_R0 + S_OFF]! @ save returned r0
  144. disable_irq_notrace @ disable interrupts
  145. ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
  146. - tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
  147. + tst r1, #((_TIF_SYSCALL_WORK | _TIF_WORK_MASK) & ~_TIF_SECCOMP)
  148. + bne do_slower_path
  149. + tst r1, #_TIF_SECCOMP
  150. beq no_work_pending
  151. +do_slower_path:
  152. UNWIND(.fnend )
  153. ENDPROC(ret_fast_syscall)
  154. diff -Nur linux-4.6.6.orig/arch/arm/kernel/process.c linux-4.6.6/arch/arm/kernel/process.c
  155. --- linux-4.6.6.orig/arch/arm/kernel/process.c 2016-08-10 12:54:57.000000000 +0200
  156. +++ linux-4.6.6/arch/arm/kernel/process.c 2016-08-22 21:27:46.313822571 +0200
  157. @@ -319,6 +319,30 @@
  158. }
  159. #ifdef CONFIG_MMU
  160. +/*
  161. + * CONFIG_SPLIT_PTLOCK_CPUS results in a page->ptl lock. If the lock is not
  162. + * initialized by pgtable_page_ctor() then a coredump of the vector page will
  163. + * fail.
  164. + */
  165. +static int __init vectors_user_mapping_init_page(void)
  166. +{
  167. + struct page *page;
  168. + unsigned long addr = 0xffff0000;
  169. + pgd_t *pgd;
  170. + pud_t *pud;
  171. + pmd_t *pmd;
  172. +
  173. + pgd = pgd_offset_k(addr);
  174. + pud = pud_offset(pgd, addr);
  175. + pmd = pmd_offset(pud, addr);
  176. + page = pmd_page(*(pmd));
  177. +
  178. + pgtable_page_ctor(page);
  179. +
  180. + return 0;
  181. +}
  182. +late_initcall(vectors_user_mapping_init_page);
  183. +
  184. #ifdef CONFIG_KUSER_HELPERS
  185. /*
  186. * The vectors page is always readable from user space for the
  187. diff -Nur linux-4.6.6.orig/arch/arm/kernel/signal.c linux-4.6.6/arch/arm/kernel/signal.c
  188. --- linux-4.6.6.orig/arch/arm/kernel/signal.c 2016-08-10 12:54:57.000000000 +0200
  189. +++ linux-4.6.6/arch/arm/kernel/signal.c 2016-08-22 21:27:46.313822571 +0200
  190. @@ -572,7 +572,8 @@
  191. */
  192. trace_hardirqs_off();
  193. do {
  194. - if (likely(thread_flags & _TIF_NEED_RESCHED)) {
  195. + if (likely(thread_flags & (_TIF_NEED_RESCHED |
  196. + _TIF_NEED_RESCHED_LAZY))) {
  197. schedule();
  198. } else {
  199. if (unlikely(!user_mode(regs)))
  200. diff -Nur linux-4.6.6.orig/arch/arm/kernel/smp.c linux-4.6.6/arch/arm/kernel/smp.c
  201. --- linux-4.6.6.orig/arch/arm/kernel/smp.c 2016-08-10 12:54:57.000000000 +0200
  202. +++ linux-4.6.6/arch/arm/kernel/smp.c 2016-08-22 21:27:46.313822571 +0200
  203. @@ -234,8 +234,6 @@
  204. flush_cache_louis();
  205. local_flush_tlb_all();
  206. - clear_tasks_mm_cpumask(cpu);
  207. -
  208. return 0;
  209. }
  210. @@ -251,6 +249,9 @@
  211. pr_err("CPU%u: cpu didn't die\n", cpu);
  212. return;
  213. }
  214. +
  215. + clear_tasks_mm_cpumask(cpu);
  216. +
  217. pr_notice("CPU%u: shutdown\n", cpu);
  218. /*
  219. diff -Nur linux-4.6.6.orig/arch/arm/kernel/unwind.c linux-4.6.6/arch/arm/kernel/unwind.c
  220. --- linux-4.6.6.orig/arch/arm/kernel/unwind.c 2016-08-10 12:54:57.000000000 +0200
  221. +++ linux-4.6.6/arch/arm/kernel/unwind.c 2016-08-22 21:27:46.317822824 +0200
  222. @@ -93,7 +93,7 @@
  223. static const struct unwind_idx *__origin_unwind_idx;
  224. extern const struct unwind_idx __stop_unwind_idx[];
  225. -static DEFINE_SPINLOCK(unwind_lock);
  226. +static DEFINE_RAW_SPINLOCK(unwind_lock);
  227. static LIST_HEAD(unwind_tables);
  228. /* Convert a prel31 symbol to an absolute address */
  229. @@ -201,7 +201,7 @@
  230. /* module unwind tables */
  231. struct unwind_table *table;
  232. - spin_lock_irqsave(&unwind_lock, flags);
  233. + raw_spin_lock_irqsave(&unwind_lock, flags);
  234. list_for_each_entry(table, &unwind_tables, list) {
  235. if (addr >= table->begin_addr &&
  236. addr < table->end_addr) {
  237. @@ -213,7 +213,7 @@
  238. break;
  239. }
  240. }
  241. - spin_unlock_irqrestore(&unwind_lock, flags);
  242. + raw_spin_unlock_irqrestore(&unwind_lock, flags);
  243. }
  244. pr_debug("%s: idx = %p\n", __func__, idx);
  245. @@ -529,9 +529,9 @@
  246. tab->begin_addr = text_addr;
  247. tab->end_addr = text_addr + text_size;
  248. - spin_lock_irqsave(&unwind_lock, flags);
  249. + raw_spin_lock_irqsave(&unwind_lock, flags);
  250. list_add_tail(&tab->list, &unwind_tables);
  251. - spin_unlock_irqrestore(&unwind_lock, flags);
  252. + raw_spin_unlock_irqrestore(&unwind_lock, flags);
  253. return tab;
  254. }
  255. @@ -543,9 +543,9 @@
  256. if (!tab)
  257. return;
  258. - spin_lock_irqsave(&unwind_lock, flags);
  259. + raw_spin_lock_irqsave(&unwind_lock, flags);
  260. list_del(&tab->list);
  261. - spin_unlock_irqrestore(&unwind_lock, flags);
  262. + raw_spin_unlock_irqrestore(&unwind_lock, flags);
  263. kfree(tab);
  264. }
  265. diff -Nur linux-4.6.6.orig/arch/arm/kvm/arm.c linux-4.6.6/arch/arm/kvm/arm.c
  266. --- linux-4.6.6.orig/arch/arm/kvm/arm.c 2016-08-10 12:54:57.000000000 +0200
  267. +++ linux-4.6.6/arch/arm/kvm/arm.c 2016-08-22 21:27:46.321823084 +0200
  268. @@ -582,7 +582,7 @@
  269. * involves poking the GIC, which must be done in a
  270. * non-preemptible context.
  271. */
  272. - preempt_disable();
  273. + migrate_disable();
  274. kvm_pmu_flush_hwstate(vcpu);
  275. kvm_timer_flush_hwstate(vcpu);
  276. kvm_vgic_flush_hwstate(vcpu);
  277. @@ -603,7 +603,7 @@
  278. kvm_pmu_sync_hwstate(vcpu);
  279. kvm_timer_sync_hwstate(vcpu);
  280. kvm_vgic_sync_hwstate(vcpu);
  281. - preempt_enable();
  282. + migrate_enable();
  283. continue;
  284. }
  285. @@ -659,7 +659,7 @@
  286. kvm_vgic_sync_hwstate(vcpu);
  287. - preempt_enable();
  288. + migrate_enable();
  289. ret = handle_exit(vcpu, run, ret);
  290. }
  291. diff -Nur linux-4.6.6.orig/arch/arm/mach-exynos/platsmp.c linux-4.6.6/arch/arm/mach-exynos/platsmp.c
  292. --- linux-4.6.6.orig/arch/arm/mach-exynos/platsmp.c 2016-08-10 12:54:57.000000000 +0200
  293. +++ linux-4.6.6/arch/arm/mach-exynos/platsmp.c 2016-08-22 21:27:46.329823597 +0200
  294. @@ -229,7 +229,7 @@
  295. return (void __iomem *)(S5P_VA_SCU);
  296. }
  297. -static DEFINE_SPINLOCK(boot_lock);
  298. +static DEFINE_RAW_SPINLOCK(boot_lock);
  299. static void exynos_secondary_init(unsigned int cpu)
  300. {
  301. @@ -242,8 +242,8 @@
  302. /*
  303. * Synchronise with the boot thread.
  304. */
  305. - spin_lock(&boot_lock);
  306. - spin_unlock(&boot_lock);
  307. + raw_spin_lock(&boot_lock);
  308. + raw_spin_unlock(&boot_lock);
  309. }
  310. int exynos_set_boot_addr(u32 core_id, unsigned long boot_addr)
  311. @@ -307,7 +307,7 @@
  312. * Set synchronisation state between this boot processor
  313. * and the secondary one
  314. */
  315. - spin_lock(&boot_lock);
  316. + raw_spin_lock(&boot_lock);
  317. /*
  318. * The secondary processor is waiting to be released from
  319. @@ -334,7 +334,7 @@
  320. if (timeout == 0) {
  321. printk(KERN_ERR "cpu1 power enable failed");
  322. - spin_unlock(&boot_lock);
  323. + raw_spin_unlock(&boot_lock);
  324. return -ETIMEDOUT;
  325. }
  326. }
  327. @@ -380,7 +380,7 @@
  328. * calibrations, then wait for it to finish
  329. */
  330. fail:
  331. - spin_unlock(&boot_lock);
  332. + raw_spin_unlock(&boot_lock);
  333. return pen_release != -1 ? ret : 0;
  334. }
  335. diff -Nur linux-4.6.6.orig/arch/arm/mach-hisi/platmcpm.c linux-4.6.6/arch/arm/mach-hisi/platmcpm.c
  336. --- linux-4.6.6.orig/arch/arm/mach-hisi/platmcpm.c 2016-08-10 12:54:57.000000000 +0200
  337. +++ linux-4.6.6/arch/arm/mach-hisi/platmcpm.c 2016-08-22 21:27:46.337824110 +0200
  338. @@ -61,7 +61,7 @@
  339. static void __iomem *sysctrl, *fabric;
  340. static int hip04_cpu_table[HIP04_MAX_CLUSTERS][HIP04_MAX_CPUS_PER_CLUSTER];
  341. -static DEFINE_SPINLOCK(boot_lock);
  342. +static DEFINE_RAW_SPINLOCK(boot_lock);
  343. static u32 fabric_phys_addr;
  344. /*
  345. * [0]: bootwrapper physical address
  346. @@ -113,7 +113,7 @@
  347. if (cluster >= HIP04_MAX_CLUSTERS || cpu >= HIP04_MAX_CPUS_PER_CLUSTER)
  348. return -EINVAL;
  349. - spin_lock_irq(&boot_lock);
  350. + raw_spin_lock_irq(&boot_lock);
  351. if (hip04_cpu_table[cluster][cpu])
  352. goto out;
  353. @@ -147,7 +147,7 @@
  354. out:
  355. hip04_cpu_table[cluster][cpu]++;
  356. - spin_unlock_irq(&boot_lock);
  357. + raw_spin_unlock_irq(&boot_lock);
  358. return 0;
  359. }
  360. @@ -162,11 +162,11 @@
  361. cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  362. cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  363. - spin_lock(&boot_lock);
  364. + raw_spin_lock(&boot_lock);
  365. hip04_cpu_table[cluster][cpu]--;
  366. if (hip04_cpu_table[cluster][cpu] == 1) {
  367. /* A power_up request went ahead of us. */
  368. - spin_unlock(&boot_lock);
  369. + raw_spin_unlock(&boot_lock);
  370. return;
  371. } else if (hip04_cpu_table[cluster][cpu] > 1) {
  372. pr_err("Cluster %d CPU%d boots multiple times\n", cluster, cpu);
  373. @@ -174,7 +174,7 @@
  374. }
  375. last_man = hip04_cluster_is_down(cluster);
  376. - spin_unlock(&boot_lock);
  377. + raw_spin_unlock(&boot_lock);
  378. if (last_man) {
  379. /* Since it's Cortex A15, disable L2 prefetching. */
  380. asm volatile(
  381. @@ -203,7 +203,7 @@
  382. cpu >= HIP04_MAX_CPUS_PER_CLUSTER);
  383. count = TIMEOUT_MSEC / POLL_MSEC;
  384. - spin_lock_irq(&boot_lock);
  385. + raw_spin_lock_irq(&boot_lock);
  386. for (tries = 0; tries < count; tries++) {
  387. if (hip04_cpu_table[cluster][cpu])
  388. goto err;
  389. @@ -211,10 +211,10 @@
  390. data = readl_relaxed(sysctrl + SC_CPU_RESET_STATUS(cluster));
  391. if (data & CORE_WFI_STATUS(cpu))
  392. break;
  393. - spin_unlock_irq(&boot_lock);
  394. + raw_spin_unlock_irq(&boot_lock);
  395. /* Wait for clean L2 when the whole cluster is down. */
  396. msleep(POLL_MSEC);
  397. - spin_lock_irq(&boot_lock);
  398. + raw_spin_lock_irq(&boot_lock);
  399. }
  400. if (tries >= count)
  401. goto err;
  402. @@ -231,10 +231,10 @@
  403. goto err;
  404. if (hip04_cluster_is_down(cluster))
  405. hip04_set_snoop_filter(cluster, 0);
  406. - spin_unlock_irq(&boot_lock);
  407. + raw_spin_unlock_irq(&boot_lock);
  408. return 1;
  409. err:
  410. - spin_unlock_irq(&boot_lock);
  411. + raw_spin_unlock_irq(&boot_lock);
  412. return 0;
  413. }
  414. #endif
  415. diff -Nur linux-4.6.6.orig/arch/arm/mach-imx/Kconfig linux-4.6.6/arch/arm/mach-imx/Kconfig
  416. --- linux-4.6.6.orig/arch/arm/mach-imx/Kconfig 2016-08-10 12:54:57.000000000 +0200
  417. +++ linux-4.6.6/arch/arm/mach-imx/Kconfig 2016-08-22 21:27:46.337824110 +0200
  418. @@ -526,7 +526,7 @@
  419. bool "i.MX6 Quad/DualLite support"
  420. select ARM_ERRATA_764369 if SMP
  421. select HAVE_ARM_SCU if SMP
  422. - select HAVE_ARM_TWD if SMP
  423. + select HAVE_ARM_TWD
  424. select PCI_DOMAINS if PCI
  425. select PINCTRL_IMX6Q
  426. select SOC_IMX6
  427. diff -Nur linux-4.6.6.orig/arch/arm/mach-omap2/omap-smp.c linux-4.6.6/arch/arm/mach-omap2/omap-smp.c
  428. --- linux-4.6.6.orig/arch/arm/mach-omap2/omap-smp.c 2016-08-10 12:54:57.000000000 +0200
  429. +++ linux-4.6.6/arch/arm/mach-omap2/omap-smp.c 2016-08-22 21:27:46.345824623 +0200
  430. @@ -43,7 +43,7 @@
  431. /* SCU base address */
  432. static void __iomem *scu_base;
  433. -static DEFINE_SPINLOCK(boot_lock);
  434. +static DEFINE_RAW_SPINLOCK(boot_lock);
  435. void __iomem *omap4_get_scu_base(void)
  436. {
  437. @@ -74,8 +74,8 @@
  438. /*
  439. * Synchronise with the boot thread.
  440. */
  441. - spin_lock(&boot_lock);
  442. - spin_unlock(&boot_lock);
  443. + raw_spin_lock(&boot_lock);
  444. + raw_spin_unlock(&boot_lock);
  445. }
  446. static int omap4_boot_secondary(unsigned int cpu, struct task_struct *idle)
  447. @@ -89,7 +89,7 @@
  448. * Set synchronisation state between this boot processor
  449. * and the secondary one
  450. */
  451. - spin_lock(&boot_lock);
  452. + raw_spin_lock(&boot_lock);
  453. /*
  454. * Update the AuxCoreBoot0 with boot state for secondary core.
  455. @@ -166,7 +166,7 @@
  456. * Now the secondary core is starting up let it run its
  457. * calibrations, then wait for it to finish
  458. */
  459. - spin_unlock(&boot_lock);
  460. + raw_spin_unlock(&boot_lock);
  461. return 0;
  462. }
  463. diff -Nur linux-4.6.6.orig/arch/arm/mach-prima2/platsmp.c linux-4.6.6/arch/arm/mach-prima2/platsmp.c
  464. --- linux-4.6.6.orig/arch/arm/mach-prima2/platsmp.c 2016-08-10 12:54:57.000000000 +0200
  465. +++ linux-4.6.6/arch/arm/mach-prima2/platsmp.c 2016-08-22 21:27:46.353825136 +0200
  466. @@ -22,7 +22,7 @@
  467. static void __iomem *clk_base;
  468. -static DEFINE_SPINLOCK(boot_lock);
  469. +static DEFINE_RAW_SPINLOCK(boot_lock);
  470. static void sirfsoc_secondary_init(unsigned int cpu)
  471. {
  472. @@ -36,8 +36,8 @@
  473. /*
  474. * Synchronise with the boot thread.
  475. */
  476. - spin_lock(&boot_lock);
  477. - spin_unlock(&boot_lock);
  478. + raw_spin_lock(&boot_lock);
  479. + raw_spin_unlock(&boot_lock);
  480. }
  481. static const struct of_device_id clk_ids[] = {
  482. @@ -75,7 +75,7 @@
  483. /* make sure write buffer is drained */
  484. mb();
  485. - spin_lock(&boot_lock);
  486. + raw_spin_lock(&boot_lock);
  487. /*
  488. * The secondary processor is waiting to be released from
  489. @@ -107,7 +107,7 @@
  490. * now the secondary core is starting up let it run its
  491. * calibrations, then wait for it to finish
  492. */
  493. - spin_unlock(&boot_lock);
  494. + raw_spin_unlock(&boot_lock);
  495. return pen_release != -1 ? -ENOSYS : 0;
  496. }
  497. diff -Nur linux-4.6.6.orig/arch/arm/mach-qcom/platsmp.c linux-4.6.6/arch/arm/mach-qcom/platsmp.c
  498. --- linux-4.6.6.orig/arch/arm/mach-qcom/platsmp.c 2016-08-10 12:54:57.000000000 +0200
  499. +++ linux-4.6.6/arch/arm/mach-qcom/platsmp.c 2016-08-22 21:27:46.361825650 +0200
  500. @@ -46,7 +46,7 @@
  501. extern void secondary_startup_arm(void);
  502. -static DEFINE_SPINLOCK(boot_lock);
  503. +static DEFINE_RAW_SPINLOCK(boot_lock);
  504. #ifdef CONFIG_HOTPLUG_CPU
  505. static void qcom_cpu_die(unsigned int cpu)
  506. @@ -60,8 +60,8 @@
  507. /*
  508. * Synchronise with the boot thread.
  509. */
  510. - spin_lock(&boot_lock);
  511. - spin_unlock(&boot_lock);
  512. + raw_spin_lock(&boot_lock);
  513. + raw_spin_unlock(&boot_lock);
  514. }
  515. static int scss_release_secondary(unsigned int cpu)
  516. @@ -284,7 +284,7 @@
  517. * set synchronisation state between this boot processor
  518. * and the secondary one
  519. */
  520. - spin_lock(&boot_lock);
  521. + raw_spin_lock(&boot_lock);
  522. /*
  523. * Send the secondary CPU a soft interrupt, thereby causing
  524. @@ -297,7 +297,7 @@
  525. * now the secondary core is starting up let it run its
  526. * calibrations, then wait for it to finish
  527. */
  528. - spin_unlock(&boot_lock);
  529. + raw_spin_unlock(&boot_lock);
  530. return ret;
  531. }
  532. diff -Nur linux-4.6.6.orig/arch/arm/mach-spear/platsmp.c linux-4.6.6/arch/arm/mach-spear/platsmp.c
  533. --- linux-4.6.6.orig/arch/arm/mach-spear/platsmp.c 2016-08-10 12:54:57.000000000 +0200
  534. +++ linux-4.6.6/arch/arm/mach-spear/platsmp.c 2016-08-22 21:27:46.365825902 +0200
  535. @@ -32,7 +32,7 @@
  536. sync_cache_w(&pen_release);
  537. }
  538. -static DEFINE_SPINLOCK(boot_lock);
  539. +static DEFINE_RAW_SPINLOCK(boot_lock);
  540. static void __iomem *scu_base = IOMEM(VA_SCU_BASE);
  541. @@ -47,8 +47,8 @@
  542. /*
  543. * Synchronise with the boot thread.
  544. */
  545. - spin_lock(&boot_lock);
  546. - spin_unlock(&boot_lock);
  547. + raw_spin_lock(&boot_lock);
  548. + raw_spin_unlock(&boot_lock);
  549. }
  550. static int spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle)
  551. @@ -59,7 +59,7 @@
  552. * set synchronisation state between this boot processor
  553. * and the secondary one
  554. */
  555. - spin_lock(&boot_lock);
  556. + raw_spin_lock(&boot_lock);
  557. /*
  558. * The secondary processor is waiting to be released from
  559. @@ -84,7 +84,7 @@
  560. * now the secondary core is starting up let it run its
  561. * calibrations, then wait for it to finish
  562. */
  563. - spin_unlock(&boot_lock);
  564. + raw_spin_unlock(&boot_lock);
  565. return pen_release != -1 ? -ENOSYS : 0;
  566. }
  567. diff -Nur linux-4.6.6.orig/arch/arm/mach-sti/platsmp.c linux-4.6.6/arch/arm/mach-sti/platsmp.c
  568. --- linux-4.6.6.orig/arch/arm/mach-sti/platsmp.c 2016-08-10 12:54:57.000000000 +0200
  569. +++ linux-4.6.6/arch/arm/mach-sti/platsmp.c 2016-08-22 21:27:46.369826163 +0200
  570. @@ -35,7 +35,7 @@
  571. sync_cache_w(&pen_release);
  572. }
  573. -static DEFINE_SPINLOCK(boot_lock);
  574. +static DEFINE_RAW_SPINLOCK(boot_lock);
  575. static void sti_secondary_init(unsigned int cpu)
  576. {
  577. @@ -48,8 +48,8 @@
  578. /*
  579. * Synchronise with the boot thread.
  580. */
  581. - spin_lock(&boot_lock);
  582. - spin_unlock(&boot_lock);
  583. + raw_spin_lock(&boot_lock);
  584. + raw_spin_unlock(&boot_lock);
  585. }
  586. static int sti_boot_secondary(unsigned int cpu, struct task_struct *idle)
  587. @@ -60,7 +60,7 @@
  588. * set synchronisation state between this boot processor
  589. * and the secondary one
  590. */
  591. - spin_lock(&boot_lock);
  592. + raw_spin_lock(&boot_lock);
  593. /*
  594. * The secondary processor is waiting to be released from
  595. @@ -91,7 +91,7 @@
  596. * now the secondary core is starting up let it run its
  597. * calibrations, then wait for it to finish
  598. */
  599. - spin_unlock(&boot_lock);
  600. + raw_spin_unlock(&boot_lock);
  601. return pen_release != -1 ? -ENOSYS : 0;
  602. }
  603. diff -Nur linux-4.6.6.orig/arch/arm/mm/fault.c linux-4.6.6/arch/arm/mm/fault.c
  604. --- linux-4.6.6.orig/arch/arm/mm/fault.c 2016-08-10 12:54:57.000000000 +0200
  605. +++ linux-4.6.6/arch/arm/mm/fault.c 2016-08-22 21:27:46.369826163 +0200
  606. @@ -430,6 +430,9 @@
  607. if (addr < TASK_SIZE)
  608. return do_page_fault(addr, fsr, regs);
  609. + if (interrupts_enabled(regs))
  610. + local_irq_enable();
  611. +
  612. if (user_mode(regs))
  613. goto bad_area;
  614. @@ -497,6 +500,9 @@
  615. static int
  616. do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
  617. {
  618. + if (interrupts_enabled(regs))
  619. + local_irq_enable();
  620. +
  621. do_bad_area(addr, fsr, regs);
  622. return 0;
  623. }
  624. diff -Nur linux-4.6.6.orig/arch/arm/mm/highmem.c linux-4.6.6/arch/arm/mm/highmem.c
  625. --- linux-4.6.6.orig/arch/arm/mm/highmem.c 2016-08-10 12:54:57.000000000 +0200
  626. +++ linux-4.6.6/arch/arm/mm/highmem.c 2016-08-22 21:27:46.369826163 +0200
  627. @@ -34,6 +34,11 @@
  628. return *ptep;
  629. }
  630. +static unsigned int fixmap_idx(int type)
  631. +{
  632. + return FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  633. +}
  634. +
  635. void *kmap(struct page *page)
  636. {
  637. might_sleep();
  638. @@ -54,12 +59,13 @@
  639. void *kmap_atomic(struct page *page)
  640. {
  641. + pte_t pte = mk_pte(page, kmap_prot);
  642. unsigned int idx;
  643. unsigned long vaddr;
  644. void *kmap;
  645. int type;
  646. - preempt_disable();
  647. + preempt_disable_nort();
  648. pagefault_disable();
  649. if (!PageHighMem(page))
  650. return page_address(page);
  651. @@ -79,7 +85,7 @@
  652. type = kmap_atomic_idx_push();
  653. - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  654. + idx = fixmap_idx(type);
  655. vaddr = __fix_to_virt(idx);
  656. #ifdef CONFIG_DEBUG_HIGHMEM
  657. /*
  658. @@ -93,7 +99,10 @@
  659. * in place, so the contained TLB flush ensures the TLB is updated
  660. * with the new mapping.
  661. */
  662. - set_fixmap_pte(idx, mk_pte(page, kmap_prot));
  663. +#ifdef CONFIG_PREEMPT_RT_FULL
  664. + current->kmap_pte[type] = pte;
  665. +#endif
  666. + set_fixmap_pte(idx, pte);
  667. return (void *)vaddr;
  668. }
  669. @@ -106,44 +115,75 @@
  670. if (kvaddr >= (void *)FIXADDR_START) {
  671. type = kmap_atomic_idx();
  672. - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  673. + idx = fixmap_idx(type);
  674. if (cache_is_vivt())
  675. __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
  676. +#ifdef CONFIG_PREEMPT_RT_FULL
  677. + current->kmap_pte[type] = __pte(0);
  678. +#endif
  679. #ifdef CONFIG_DEBUG_HIGHMEM
  680. BUG_ON(vaddr != __fix_to_virt(idx));
  681. - set_fixmap_pte(idx, __pte(0));
  682. #else
  683. (void) idx; /* to kill a warning */
  684. #endif
  685. + set_fixmap_pte(idx, __pte(0));
  686. kmap_atomic_idx_pop();
  687. } else if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) {
  688. /* this address was obtained through kmap_high_get() */
  689. kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)]));
  690. }
  691. pagefault_enable();
  692. - preempt_enable();
  693. + preempt_enable_nort();
  694. }
  695. EXPORT_SYMBOL(__kunmap_atomic);
  696. void *kmap_atomic_pfn(unsigned long pfn)
  697. {
  698. + pte_t pte = pfn_pte(pfn, kmap_prot);
  699. unsigned long vaddr;
  700. int idx, type;
  701. struct page *page = pfn_to_page(pfn);
  702. - preempt_disable();
  703. + preempt_disable_nort();
  704. pagefault_disable();
  705. if (!PageHighMem(page))
  706. return page_address(page);
  707. type = kmap_atomic_idx_push();
  708. - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  709. + idx = fixmap_idx(type);
  710. vaddr = __fix_to_virt(idx);
  711. #ifdef CONFIG_DEBUG_HIGHMEM
  712. BUG_ON(!pte_none(get_fixmap_pte(vaddr)));
  713. #endif
  714. - set_fixmap_pte(idx, pfn_pte(pfn, kmap_prot));
  715. +#ifdef CONFIG_PREEMPT_RT_FULL
  716. + current->kmap_pte[type] = pte;
  717. +#endif
  718. + set_fixmap_pte(idx, pte);
  719. return (void *)vaddr;
  720. }
  721. +#if defined CONFIG_PREEMPT_RT_FULL
  722. +void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p)
  723. +{
  724. + int i;
  725. +
  726. + /*
  727. + * Clear @prev's kmap_atomic mappings
  728. + */
  729. + for (i = 0; i < prev_p->kmap_idx; i++) {
  730. + int idx = fixmap_idx(i);
  731. +
  732. + set_fixmap_pte(idx, __pte(0));
  733. + }
  734. + /*
  735. + * Restore @next_p's kmap_atomic mappings
  736. + */
  737. + for (i = 0; i < next_p->kmap_idx; i++) {
  738. + int idx = fixmap_idx(i);
  739. +
  740. + if (!pte_none(next_p->kmap_pte[i]))
  741. + set_fixmap_pte(idx, next_p->kmap_pte[i]);
  742. + }
  743. +}
  744. +#endif
  745. diff -Nur linux-4.6.6.orig/arch/arm/plat-versatile/platsmp.c linux-4.6.6/arch/arm/plat-versatile/platsmp.c
  746. --- linux-4.6.6.orig/arch/arm/plat-versatile/platsmp.c 2016-08-10 12:54:57.000000000 +0200
  747. +++ linux-4.6.6/arch/arm/plat-versatile/platsmp.c 2016-08-22 21:27:46.381826929 +0200
  748. @@ -30,7 +30,7 @@
  749. sync_cache_w(&pen_release);
  750. }
  751. -static DEFINE_SPINLOCK(boot_lock);
  752. +static DEFINE_RAW_SPINLOCK(boot_lock);
  753. void versatile_secondary_init(unsigned int cpu)
  754. {
  755. @@ -43,8 +43,8 @@
  756. /*
  757. * Synchronise with the boot thread.
  758. */
  759. - spin_lock(&boot_lock);
  760. - spin_unlock(&boot_lock);
  761. + raw_spin_lock(&boot_lock);
  762. + raw_spin_unlock(&boot_lock);
  763. }
  764. int versatile_boot_secondary(unsigned int cpu, struct task_struct *idle)
  765. @@ -55,7 +55,7 @@
  766. * Set synchronisation state between this boot processor
  767. * and the secondary one
  768. */
  769. - spin_lock(&boot_lock);
  770. + raw_spin_lock(&boot_lock);
  771. /*
  772. * This is really belt and braces; we hold unintended secondary
  773. @@ -85,7 +85,7 @@
  774. * now the secondary core is starting up let it run its
  775. * calibrations, then wait for it to finish
  776. */
  777. - spin_unlock(&boot_lock);
  778. + raw_spin_unlock(&boot_lock);
  779. return pen_release != -1 ? -ENOSYS : 0;
  780. }
  781. diff -Nur linux-4.6.6.orig/arch/arm64/include/asm/thread_info.h linux-4.6.6/arch/arm64/include/asm/thread_info.h
  782. --- linux-4.6.6.orig/arch/arm64/include/asm/thread_info.h 2016-08-10 12:54:57.000000000 +0200
  783. +++ linux-4.6.6/arch/arm64/include/asm/thread_info.h 2016-08-22 21:27:46.393827702 +0200
  784. @@ -49,6 +49,7 @@
  785. mm_segment_t addr_limit; /* address limit */
  786. struct task_struct *task; /* main task structure */
  787. int preempt_count; /* 0 => preemptable, <0 => bug */
  788. + int preempt_lazy_count; /* 0 => preemptable, <0 => bug */
  789. int cpu; /* cpu */
  790. };
  791. @@ -109,6 +110,7 @@
  792. #define TIF_NEED_RESCHED 1
  793. #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */
  794. #define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */
  795. +#define TIF_NEED_RESCHED_LAZY 4
  796. #define TIF_NOHZ 7
  797. #define TIF_SYSCALL_TRACE 8
  798. #define TIF_SYSCALL_AUDIT 9
  799. @@ -124,6 +126,7 @@
  800. #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
  801. #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
  802. #define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
  803. +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
  804. #define _TIF_NOHZ (1 << TIF_NOHZ)
  805. #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
  806. #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
  807. @@ -132,7 +135,8 @@
  808. #define _TIF_32BIT (1 << TIF_32BIT)
  809. #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
  810. - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE)
  811. + _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
  812. + _TIF_NEED_RESCHED_LAZY)
  813. #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
  814. _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
  815. diff -Nur linux-4.6.6.orig/arch/arm64/Kconfig linux-4.6.6/arch/arm64/Kconfig
  816. --- linux-4.6.6.orig/arch/arm64/Kconfig 2016-08-10 12:54:57.000000000 +0200
  817. +++ linux-4.6.6/arch/arm64/Kconfig 2016-08-22 21:27:46.389827442 +0200
  818. @@ -81,6 +81,7 @@
  819. select HAVE_PERF_REGS
  820. select HAVE_PERF_USER_STACK_DUMP
  821. select HAVE_RCU_TABLE_FREE
  822. + select HAVE_PREEMPT_LAZY
  823. select HAVE_SYSCALL_TRACEPOINTS
  824. select IOMMU_DMA if IOMMU_SUPPORT
  825. select IRQ_DOMAIN
  826. @@ -624,7 +625,7 @@
  827. config XEN
  828. bool "Xen guest support on ARM64"
  829. - depends on ARM64 && OF
  830. + depends on ARM64 && OF && !PREEMPT_RT_FULL
  831. select SWIOTLB_XEN
  832. select PARAVIRT
  833. help
  834. diff -Nur linux-4.6.6.orig/arch/arm64/kernel/asm-offsets.c linux-4.6.6/arch/arm64/kernel/asm-offsets.c
  835. --- linux-4.6.6.orig/arch/arm64/kernel/asm-offsets.c 2016-08-10 12:54:57.000000000 +0200
  836. +++ linux-4.6.6/arch/arm64/kernel/asm-offsets.c 2016-08-22 21:27:46.401828215 +0200
  837. @@ -36,6 +36,7 @@
  838. BLANK();
  839. DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
  840. DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
  841. + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count));
  842. DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit));
  843. DEFINE(TI_TASK, offsetof(struct thread_info, task));
  844. DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
  845. diff -Nur linux-4.6.6.orig/arch/arm64/kernel/entry.S linux-4.6.6/arch/arm64/kernel/entry.S
  846. --- linux-4.6.6.orig/arch/arm64/kernel/entry.S 2016-08-10 12:54:57.000000000 +0200
  847. +++ linux-4.6.6/arch/arm64/kernel/entry.S 2016-08-22 21:27:46.405828468 +0200
  848. @@ -426,11 +426,16 @@
  849. #ifdef CONFIG_PREEMPT
  850. ldr w24, [tsk, #TI_PREEMPT] // get preempt count
  851. - cbnz w24, 1f // preempt count != 0
  852. + cbnz w24, 2f // preempt count != 0
  853. ldr x0, [tsk, #TI_FLAGS] // get flags
  854. - tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
  855. - bl el1_preempt
  856. + tbnz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
  857. +
  858. + ldr w24, [tsk, #TI_PREEMPT_LAZY] // get preempt lazy count
  859. + cbnz w24, 2f // preempt lazy count != 0
  860. + tbz x0, #TIF_NEED_RESCHED_LAZY, 2f // needs rescheduling?
  861. 1:
  862. + bl el1_preempt
  863. +2:
  864. #endif
  865. #ifdef CONFIG_TRACE_IRQFLAGS
  866. bl trace_hardirqs_on
  867. @@ -444,6 +449,7 @@
  868. 1: bl preempt_schedule_irq // irq en/disable is done inside
  869. ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS
  870. tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling?
  871. + tbnz x0, #TIF_NEED_RESCHED_LAZY, 1b // needs rescheduling?
  872. ret x24
  873. #endif
  874. @@ -690,6 +696,7 @@
  875. */
  876. work_pending:
  877. tbnz x1, #TIF_NEED_RESCHED, work_resched
  878. + tbnz x1, #TIF_NEED_RESCHED_LAZY, work_resched
  879. /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
  880. mov x0, sp // 'regs'
  881. enable_irq // enable interrupts for do_notify_resume()
  882. diff -Nur linux-4.6.6.orig/arch/Kconfig linux-4.6.6/arch/Kconfig
  883. --- linux-4.6.6.orig/arch/Kconfig 2016-08-10 12:54:57.000000000 +0200
  884. +++ linux-4.6.6/arch/Kconfig 2016-08-22 21:27:46.245818207 +0200
  885. @@ -9,6 +9,7 @@
  886. tristate "OProfile system profiling"
  887. depends on PROFILING
  888. depends on HAVE_OPROFILE
  889. + depends on !PREEMPT_RT_FULL
  890. select RING_BUFFER
  891. select RING_BUFFER_ALLOW_SWAP
  892. help
  893. @@ -52,6 +53,7 @@
  894. config JUMP_LABEL
  895. bool "Optimize very unlikely/likely branches"
  896. depends on HAVE_ARCH_JUMP_LABEL
  897. + depends on (!INTERRUPT_OFF_HIST && !PREEMPT_OFF_HIST && !WAKEUP_LATENCY_HIST && !MISSED_TIMER_OFFSETS_HIST)
  898. help
  899. This option enables a transparent branch optimization that
  900. makes certain almost-always-true or almost-always-false branch
  901. diff -Nur linux-4.6.6.orig/arch/mips/Kconfig linux-4.6.6/arch/mips/Kconfig
  902. --- linux-4.6.6.orig/arch/mips/Kconfig 2016-08-10 12:54:57.000000000 +0200
  903. +++ linux-4.6.6/arch/mips/Kconfig 2016-08-22 21:27:46.417829241 +0200
  904. @@ -2416,7 +2416,7 @@
  905. #
  906. config HIGHMEM
  907. bool "High Memory Support"
  908. - depends on 32BIT && CPU_SUPPORTS_HIGHMEM && SYS_SUPPORTS_HIGHMEM && !CPU_MIPS32_3_5_EVA
  909. + depends on 32BIT && CPU_SUPPORTS_HIGHMEM && SYS_SUPPORTS_HIGHMEM && !CPU_MIPS32_3_5_EVA && !PREEMPT_RT_FULL
  910. config CPU_SUPPORTS_HIGHMEM
  911. bool
  912. diff -Nur linux-4.6.6.orig/arch/powerpc/include/asm/thread_info.h linux-4.6.6/arch/powerpc/include/asm/thread_info.h
  913. --- linux-4.6.6.orig/arch/powerpc/include/asm/thread_info.h 2016-08-10 12:54:57.000000000 +0200
  914. +++ linux-4.6.6/arch/powerpc/include/asm/thread_info.h 2016-08-22 21:27:46.433830268 +0200
  915. @@ -42,6 +42,8 @@
  916. int cpu; /* cpu we're on */
  917. int preempt_count; /* 0 => preemptable,
  918. <0 => BUG */
  919. + int preempt_lazy_count; /* 0 => preemptable,
  920. + <0 => BUG */
  921. unsigned long local_flags; /* private flags for thread */
  922. /* low level flags - has atomic operations done on it */
  923. @@ -82,8 +84,7 @@
  924. #define TIF_SYSCALL_TRACE 0 /* syscall trace active */
  925. #define TIF_SIGPENDING 1 /* signal pending */
  926. #define TIF_NEED_RESCHED 2 /* rescheduling necessary */
  927. -#define TIF_POLLING_NRFLAG 3 /* true if poll_idle() is polling
  928. - TIF_NEED_RESCHED */
  929. +#define TIF_NEED_RESCHED_LAZY 3 /* lazy rescheduling necessary */
  930. #define TIF_32BIT 4 /* 32 bit binary */
  931. #define TIF_RESTORE_TM 5 /* need to restore TM FP/VEC/VSX */
  932. #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
  933. @@ -101,6 +102,8 @@
  934. #if defined(CONFIG_PPC64)
  935. #define TIF_ELF2ABI 18 /* function descriptors must die! */
  936. #endif
  937. +#define TIF_POLLING_NRFLAG 19 /* true if poll_idle() is polling
  938. + TIF_NEED_RESCHED */
  939. /* as above, but as bit values */
  940. #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
  941. @@ -119,14 +122,16 @@
  942. #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
  943. #define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE)
  944. #define _TIF_NOHZ (1<<TIF_NOHZ)
  945. +#define _TIF_NEED_RESCHED_LAZY (1<<TIF_NEED_RESCHED_LAZY)
  946. #define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
  947. _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \
  948. _TIF_NOHZ)
  949. #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
  950. _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
  951. - _TIF_RESTORE_TM)
  952. + _TIF_RESTORE_TM | _TIF_NEED_RESCHED_LAZY)
  953. #define _TIF_PERSYSCALL_MASK (_TIF_RESTOREALL|_TIF_NOERROR)
  954. +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
  955. /* Bits in local_flags */
  956. /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */
  957. diff -Nur linux-4.6.6.orig/arch/powerpc/Kconfig linux-4.6.6/arch/powerpc/Kconfig
  958. --- linux-4.6.6.orig/arch/powerpc/Kconfig 2016-08-10 12:54:57.000000000 +0200
  959. +++ linux-4.6.6/arch/powerpc/Kconfig 2016-08-22 21:27:46.425829754 +0200
  960. @@ -57,10 +57,11 @@
  961. config RWSEM_GENERIC_SPINLOCK
  962. bool
  963. + default y if PREEMPT_RT_FULL
  964. config RWSEM_XCHGADD_ALGORITHM
  965. bool
  966. - default y
  967. + default y if !PREEMPT_RT_FULL
  968. config GENERIC_LOCKBREAK
  969. bool
  970. @@ -138,6 +139,7 @@
  971. select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
  972. select GENERIC_STRNCPY_FROM_USER
  973. select GENERIC_STRNLEN_USER
  974. + select HAVE_PREEMPT_LAZY
  975. select HAVE_MOD_ARCH_SPECIFIC
  976. select MODULES_USE_ELF_RELA
  977. select CLONE_BACKWARDS
  978. @@ -319,7 +321,7 @@
  979. config HIGHMEM
  980. bool "High memory support"
  981. - depends on PPC32
  982. + depends on PPC32 && !PREEMPT_RT_FULL
  983. source kernel/Kconfig.hz
  984. source kernel/Kconfig.preempt
  985. diff -Nur linux-4.6.6.orig/arch/powerpc/kernel/asm-offsets.c linux-4.6.6/arch/powerpc/kernel/asm-offsets.c
  986. --- linux-4.6.6.orig/arch/powerpc/kernel/asm-offsets.c 2016-08-10 12:54:57.000000000 +0200
  987. +++ linux-4.6.6/arch/powerpc/kernel/asm-offsets.c 2016-08-22 21:27:46.441830777 +0200
  988. @@ -162,6 +162,7 @@
  989. DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
  990. DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
  991. DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
  992. + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count));
  993. DEFINE(TI_TASK, offsetof(struct thread_info, task));
  994. DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
  995. diff -Nur linux-4.6.6.orig/arch/powerpc/kernel/entry_32.S linux-4.6.6/arch/powerpc/kernel/entry_32.S
  996. --- linux-4.6.6.orig/arch/powerpc/kernel/entry_32.S 2016-08-10 12:54:57.000000000 +0200
  997. +++ linux-4.6.6/arch/powerpc/kernel/entry_32.S 2016-08-22 21:27:46.481833341 +0200
  998. @@ -818,7 +818,14 @@
  999. cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
  1000. bne restore
  1001. andi. r8,r8,_TIF_NEED_RESCHED
  1002. + bne+ 1f
  1003. + lwz r0,TI_PREEMPT_LAZY(r9)
  1004. + cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
  1005. + bne restore
  1006. + lwz r0,TI_FLAGS(r9)
  1007. + andi. r0,r0,_TIF_NEED_RESCHED_LAZY
  1008. beq+ restore
  1009. +1:
  1010. lwz r3,_MSR(r1)
  1011. andi. r0,r3,MSR_EE /* interrupts off? */
  1012. beq restore /* don't schedule if so */
  1013. @@ -829,11 +836,11 @@
  1014. */
  1015. bl trace_hardirqs_off
  1016. #endif
  1017. -1: bl preempt_schedule_irq
  1018. +2: bl preempt_schedule_irq
  1019. CURRENT_THREAD_INFO(r9, r1)
  1020. lwz r3,TI_FLAGS(r9)
  1021. - andi. r0,r3,_TIF_NEED_RESCHED
  1022. - bne- 1b
  1023. + andi. r0,r3,_TIF_NEED_RESCHED_MASK
  1024. + bne- 2b
  1025. #ifdef CONFIG_TRACE_IRQFLAGS
  1026. /* And now, to properly rebalance the above, we tell lockdep they
  1027. * are being turned back on, which will happen when we return
  1028. @@ -1154,7 +1161,7 @@
  1029. #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
  1030. do_work: /* r10 contains MSR_KERNEL here */
  1031. - andi. r0,r9,_TIF_NEED_RESCHED
  1032. + andi. r0,r9,_TIF_NEED_RESCHED_MASK
  1033. beq do_user_signal
  1034. do_resched: /* r10 contains MSR_KERNEL here */
  1035. @@ -1175,7 +1182,7 @@
  1036. MTMSRD(r10) /* disable interrupts */
  1037. CURRENT_THREAD_INFO(r9, r1)
  1038. lwz r9,TI_FLAGS(r9)
  1039. - andi. r0,r9,_TIF_NEED_RESCHED
  1040. + andi. r0,r9,_TIF_NEED_RESCHED_MASK
  1041. bne- do_resched
  1042. andi. r0,r9,_TIF_USER_WORK_MASK
  1043. beq restore_user
  1044. diff -Nur linux-4.6.6.orig/arch/powerpc/kernel/entry_64.S linux-4.6.6/arch/powerpc/kernel/entry_64.S
  1045. --- linux-4.6.6.orig/arch/powerpc/kernel/entry_64.S 2016-08-10 12:54:57.000000000 +0200
  1046. +++ linux-4.6.6/arch/powerpc/kernel/entry_64.S 2016-08-22 21:27:46.485833601 +0200
  1047. @@ -644,7 +644,7 @@
  1048. bl restore_math
  1049. b restore
  1050. #endif
  1051. -1: andi. r0,r4,_TIF_NEED_RESCHED
  1052. +1: andi. r0,r4,_TIF_NEED_RESCHED_MASK
  1053. beq 2f
  1054. bl restore_interrupts
  1055. SCHEDULE_USER
  1056. @@ -706,10 +706,18 @@
  1057. #ifdef CONFIG_PREEMPT
  1058. /* Check if we need to preempt */
  1059. + lwz r8,TI_PREEMPT(r9)
  1060. + cmpwi 0,r8,0 /* if non-zero, just restore regs and return */
  1061. + bne restore
  1062. andi. r0,r4,_TIF_NEED_RESCHED
  1063. + bne+ check_count
  1064. +
  1065. + andi. r0,r4,_TIF_NEED_RESCHED_LAZY
  1066. beq+ restore
  1067. + lwz r8,TI_PREEMPT_LAZY(r9)
  1068. +
  1069. /* Check that preempt_count() == 0 and interrupts are enabled */
  1070. - lwz r8,TI_PREEMPT(r9)
  1071. +check_count:
  1072. cmpwi cr1,r8,0
  1073. ld r0,SOFTE(r1)
  1074. cmpdi r0,0
  1075. @@ -726,7 +734,7 @@
  1076. /* Re-test flags and eventually loop */
  1077. CURRENT_THREAD_INFO(r9, r1)
  1078. ld r4,TI_FLAGS(r9)
  1079. - andi. r0,r4,_TIF_NEED_RESCHED
  1080. + andi. r0,r4,_TIF_NEED_RESCHED_MASK
  1081. bne 1b
  1082. /*
  1083. diff -Nur linux-4.6.6.orig/arch/powerpc/kernel/irq.c linux-4.6.6/arch/powerpc/kernel/irq.c
  1084. --- linux-4.6.6.orig/arch/powerpc/kernel/irq.c 2016-08-10 12:54:57.000000000 +0200
  1085. +++ linux-4.6.6/arch/powerpc/kernel/irq.c 2016-08-22 21:27:46.493834114 +0200
  1086. @@ -614,6 +614,7 @@
  1087. }
  1088. }
  1089. +#ifndef CONFIG_PREEMPT_RT_FULL
  1090. void do_softirq_own_stack(void)
  1091. {
  1092. struct thread_info *curtp, *irqtp;
  1093. @@ -631,6 +632,7 @@
  1094. if (irqtp->flags)
  1095. set_bits(irqtp->flags, &curtp->flags);
  1096. }
  1097. +#endif
  1098. irq_hw_number_t virq_to_hw(unsigned int virq)
  1099. {
  1100. diff -Nur linux-4.6.6.orig/arch/powerpc/kernel/misc_32.S linux-4.6.6/arch/powerpc/kernel/misc_32.S
  1101. --- linux-4.6.6.orig/arch/powerpc/kernel/misc_32.S 2016-08-10 12:54:57.000000000 +0200
  1102. +++ linux-4.6.6/arch/powerpc/kernel/misc_32.S 2016-08-22 21:27:46.497834367 +0200
  1103. @@ -40,6 +40,7 @@
  1104. * We store the saved ksp_limit in the unused part
  1105. * of the STACK_FRAME_OVERHEAD
  1106. */
  1107. +#ifndef CONFIG_PREEMPT_RT_FULL
  1108. _GLOBAL(call_do_softirq)
  1109. mflr r0
  1110. stw r0,4(r1)
  1111. @@ -56,6 +57,7 @@
  1112. stw r10,THREAD+KSP_LIMIT(r2)
  1113. mtlr r0
  1114. blr
  1115. +#endif
  1116. /*
  1117. * void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp);
  1118. diff -Nur linux-4.6.6.orig/arch/powerpc/kernel/misc_64.S linux-4.6.6/arch/powerpc/kernel/misc_64.S
  1119. --- linux-4.6.6.orig/arch/powerpc/kernel/misc_64.S 2016-08-10 12:54:57.000000000 +0200
  1120. +++ linux-4.6.6/arch/powerpc/kernel/misc_64.S 2016-08-22 21:27:46.497834367 +0200
  1121. @@ -30,6 +30,7 @@
  1122. .text
  1123. +#ifndef CONFIG_PREEMPT_RT_FULL
  1124. _GLOBAL(call_do_softirq)
  1125. mflr r0
  1126. std r0,16(r1)
  1127. @@ -40,6 +41,7 @@
  1128. ld r0,16(r1)
  1129. mtlr r0
  1130. blr
  1131. +#endif
  1132. _GLOBAL(call_do_irq)
  1133. mflr r0
  1134. diff -Nur linux-4.6.6.orig/arch/powerpc/kvm/Kconfig linux-4.6.6/arch/powerpc/kvm/Kconfig
  1135. --- linux-4.6.6.orig/arch/powerpc/kvm/Kconfig 2016-08-10 12:54:57.000000000 +0200
  1136. +++ linux-4.6.6/arch/powerpc/kvm/Kconfig 2016-08-22 21:27:46.501834627 +0200
  1137. @@ -172,6 +172,7 @@
  1138. config KVM_MPIC
  1139. bool "KVM in-kernel MPIC emulation"
  1140. depends on KVM && E500
  1141. + depends on !PREEMPT_RT_FULL
  1142. select HAVE_KVM_IRQCHIP
  1143. select HAVE_KVM_IRQFD
  1144. select HAVE_KVM_IRQ_ROUTING
  1145. diff -Nur linux-4.6.6.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.6.6/arch/powerpc/platforms/ps3/device-init.c
  1146. --- linux-4.6.6.orig/arch/powerpc/platforms/ps3/device-init.c 2016-08-10 12:54:57.000000000 +0200
  1147. +++ linux-4.6.6/arch/powerpc/platforms/ps3/device-init.c 2016-08-22 21:27:46.509835141 +0200
  1148. @@ -752,7 +752,7 @@
  1149. }
  1150. pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op);
  1151. - res = wait_event_interruptible(dev->done.wait,
  1152. + res = swait_event_interruptible(dev->done.wait,
  1153. dev->done.done || kthread_should_stop());
  1154. if (kthread_should_stop())
  1155. res = -EINTR;
  1156. diff -Nur linux-4.6.6.orig/arch/sh/kernel/irq.c linux-4.6.6/arch/sh/kernel/irq.c
  1157. --- linux-4.6.6.orig/arch/sh/kernel/irq.c 2016-08-10 12:54:57.000000000 +0200
  1158. +++ linux-4.6.6/arch/sh/kernel/irq.c 2016-08-22 21:27:46.517835654 +0200
  1159. @@ -147,6 +147,7 @@
  1160. hardirq_ctx[cpu] = NULL;
  1161. }
  1162. +#ifndef CONFIG_PREEMPT_RT_FULL
  1163. void do_softirq_own_stack(void)
  1164. {
  1165. struct thread_info *curctx;
  1166. @@ -174,6 +175,7 @@
  1167. "r5", "r6", "r7", "r8", "r9", "r15", "t", "pr"
  1168. );
  1169. }
  1170. +#endif
  1171. #else
  1172. static inline void handle_one_irq(unsigned int irq)
  1173. {
  1174. diff -Nur linux-4.6.6.orig/arch/sparc/Kconfig linux-4.6.6/arch/sparc/Kconfig
  1175. --- linux-4.6.6.orig/arch/sparc/Kconfig 2016-08-10 12:54:57.000000000 +0200
  1176. +++ linux-4.6.6/arch/sparc/Kconfig 2016-08-22 21:27:46.525836167 +0200
  1177. @@ -184,12 +184,10 @@
  1178. source kernel/Kconfig.hz
  1179. config RWSEM_GENERIC_SPINLOCK
  1180. - bool
  1181. - default y if SPARC32
  1182. + def_bool PREEMPT_RT_FULL
  1183. config RWSEM_XCHGADD_ALGORITHM
  1184. - bool
  1185. - default y if SPARC64
  1186. + def_bool !RWSEM_GENERIC_SPINLOCK && !PREEMPT_RT_FULL
  1187. config GENERIC_HWEIGHT
  1188. bool
  1189. diff -Nur linux-4.6.6.orig/arch/sparc/kernel/irq_64.c linux-4.6.6/arch/sparc/kernel/irq_64.c
  1190. --- linux-4.6.6.orig/arch/sparc/kernel/irq_64.c 2016-08-10 12:54:57.000000000 +0200
  1191. +++ linux-4.6.6/arch/sparc/kernel/irq_64.c 2016-08-22 21:27:46.533836680 +0200
  1192. @@ -854,6 +854,7 @@
  1193. set_irq_regs(old_regs);
  1194. }
  1195. +#ifndef CONFIG_PREEMPT_RT_FULL
  1196. void do_softirq_own_stack(void)
  1197. {
  1198. void *orig_sp, *sp = softirq_stack[smp_processor_id()];
  1199. @@ -868,6 +869,7 @@
  1200. __asm__ __volatile__("mov %0, %%sp"
  1201. : : "r" (orig_sp));
  1202. }
  1203. +#endif
  1204. #ifdef CONFIG_HOTPLUG_CPU
  1205. void fixup_irqs(void)
  1206. diff -Nur linux-4.6.6.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.6.6/arch/x86/crypto/aesni-intel_glue.c
  1207. --- linux-4.6.6.orig/arch/x86/crypto/aesni-intel_glue.c 2016-08-10 12:54:57.000000000 +0200
  1208. +++ linux-4.6.6/arch/x86/crypto/aesni-intel_glue.c 2016-08-22 21:27:46.549837706 +0200
  1209. @@ -383,14 +383,14 @@
  1210. err = blkcipher_walk_virt(desc, &walk);
  1211. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1212. - kernel_fpu_begin();
  1213. while ((nbytes = walk.nbytes)) {
  1214. + kernel_fpu_begin();
  1215. aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1216. - nbytes & AES_BLOCK_MASK);
  1217. + nbytes & AES_BLOCK_MASK);
  1218. + kernel_fpu_end();
  1219. nbytes &= AES_BLOCK_SIZE - 1;
  1220. err = blkcipher_walk_done(desc, &walk, nbytes);
  1221. }
  1222. - kernel_fpu_end();
  1223. return err;
  1224. }
  1225. @@ -407,14 +407,14 @@
  1226. err = blkcipher_walk_virt(desc, &walk);
  1227. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1228. - kernel_fpu_begin();
  1229. while ((nbytes = walk.nbytes)) {
  1230. + kernel_fpu_begin();
  1231. aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1232. nbytes & AES_BLOCK_MASK);
  1233. + kernel_fpu_end();
  1234. nbytes &= AES_BLOCK_SIZE - 1;
  1235. err = blkcipher_walk_done(desc, &walk, nbytes);
  1236. }
  1237. - kernel_fpu_end();
  1238. return err;
  1239. }
  1240. @@ -431,14 +431,14 @@
  1241. err = blkcipher_walk_virt(desc, &walk);
  1242. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1243. - kernel_fpu_begin();
  1244. while ((nbytes = walk.nbytes)) {
  1245. + kernel_fpu_begin();
  1246. aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1247. nbytes & AES_BLOCK_MASK, walk.iv);
  1248. + kernel_fpu_end();
  1249. nbytes &= AES_BLOCK_SIZE - 1;
  1250. err = blkcipher_walk_done(desc, &walk, nbytes);
  1251. }
  1252. - kernel_fpu_end();
  1253. return err;
  1254. }
  1255. @@ -455,14 +455,14 @@
  1256. err = blkcipher_walk_virt(desc, &walk);
  1257. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1258. - kernel_fpu_begin();
  1259. while ((nbytes = walk.nbytes)) {
  1260. + kernel_fpu_begin();
  1261. aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1262. nbytes & AES_BLOCK_MASK, walk.iv);
  1263. + kernel_fpu_end();
  1264. nbytes &= AES_BLOCK_SIZE - 1;
  1265. err = blkcipher_walk_done(desc, &walk, nbytes);
  1266. }
  1267. - kernel_fpu_end();
  1268. return err;
  1269. }
  1270. @@ -514,18 +514,20 @@
  1271. err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
  1272. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1273. - kernel_fpu_begin();
  1274. while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
  1275. + kernel_fpu_begin();
  1276. aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1277. nbytes & AES_BLOCK_MASK, walk.iv);
  1278. + kernel_fpu_end();
  1279. nbytes &= AES_BLOCK_SIZE - 1;
  1280. err = blkcipher_walk_done(desc, &walk, nbytes);
  1281. }
  1282. if (walk.nbytes) {
  1283. + kernel_fpu_begin();
  1284. ctr_crypt_final(ctx, &walk);
  1285. + kernel_fpu_end();
  1286. err = blkcipher_walk_done(desc, &walk, 0);
  1287. }
  1288. - kernel_fpu_end();
  1289. return err;
  1290. }
  1291. diff -Nur linux-4.6.6.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.6.6/arch/x86/crypto/cast5_avx_glue.c
  1292. --- linux-4.6.6.orig/arch/x86/crypto/cast5_avx_glue.c 2016-08-10 12:54:57.000000000 +0200
  1293. +++ linux-4.6.6/arch/x86/crypto/cast5_avx_glue.c 2016-08-22 21:27:46.561838472 +0200
  1294. @@ -59,7 +59,7 @@
  1295. static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  1296. bool enc)
  1297. {
  1298. - bool fpu_enabled = false;
  1299. + bool fpu_enabled;
  1300. struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  1301. const unsigned int bsize = CAST5_BLOCK_SIZE;
  1302. unsigned int nbytes;
  1303. @@ -75,7 +75,7 @@
  1304. u8 *wsrc = walk->src.virt.addr;
  1305. u8 *wdst = walk->dst.virt.addr;
  1306. - fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
  1307. + fpu_enabled = cast5_fpu_begin(false, nbytes);
  1308. /* Process multi-block batch */
  1309. if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
  1310. @@ -103,10 +103,9 @@
  1311. } while (nbytes >= bsize);
  1312. done:
  1313. + cast5_fpu_end(fpu_enabled);
  1314. err = blkcipher_walk_done(desc, walk, nbytes);
  1315. }
  1316. -
  1317. - cast5_fpu_end(fpu_enabled);
  1318. return err;
  1319. }
  1320. @@ -227,7 +226,7 @@
  1321. static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  1322. struct scatterlist *src, unsigned int nbytes)
  1323. {
  1324. - bool fpu_enabled = false;
  1325. + bool fpu_enabled;
  1326. struct blkcipher_walk walk;
  1327. int err;
  1328. @@ -236,12 +235,11 @@
  1329. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1330. while ((nbytes = walk.nbytes)) {
  1331. - fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
  1332. + fpu_enabled = cast5_fpu_begin(false, nbytes);
  1333. nbytes = __cbc_decrypt(desc, &walk);
  1334. + cast5_fpu_end(fpu_enabled);
  1335. err = blkcipher_walk_done(desc, &walk, nbytes);
  1336. }
  1337. -
  1338. - cast5_fpu_end(fpu_enabled);
  1339. return err;
  1340. }
  1341. @@ -311,7 +309,7 @@
  1342. static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  1343. struct scatterlist *src, unsigned int nbytes)
  1344. {
  1345. - bool fpu_enabled = false;
  1346. + bool fpu_enabled;
  1347. struct blkcipher_walk walk;
  1348. int err;
  1349. @@ -320,13 +318,12 @@
  1350. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1351. while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
  1352. - fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
  1353. + fpu_enabled = cast5_fpu_begin(false, nbytes);
  1354. nbytes = __ctr_crypt(desc, &walk);
  1355. + cast5_fpu_end(fpu_enabled);
  1356. err = blkcipher_walk_done(desc, &walk, nbytes);
  1357. }
  1358. - cast5_fpu_end(fpu_enabled);
  1359. -
  1360. if (walk.nbytes) {
  1361. ctr_crypt_final(desc, &walk);
  1362. err = blkcipher_walk_done(desc, &walk, 0);
  1363. diff -Nur linux-4.6.6.orig/arch/x86/crypto/glue_helper.c linux-4.6.6/arch/x86/crypto/glue_helper.c
  1364. --- linux-4.6.6.orig/arch/x86/crypto/glue_helper.c 2016-08-10 12:54:57.000000000 +0200
  1365. +++ linux-4.6.6/arch/x86/crypto/glue_helper.c 2016-08-22 21:27:46.565838732 +0200
  1366. @@ -39,7 +39,7 @@
  1367. void *ctx = crypto_blkcipher_ctx(desc->tfm);
  1368. const unsigned int bsize = 128 / 8;
  1369. unsigned int nbytes, i, func_bytes;
  1370. - bool fpu_enabled = false;
  1371. + bool fpu_enabled;
  1372. int err;
  1373. err = blkcipher_walk_virt(desc, walk);
  1374. @@ -49,7 +49,7 @@
  1375. u8 *wdst = walk->dst.virt.addr;
  1376. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1377. - desc, fpu_enabled, nbytes);
  1378. + desc, false, nbytes);
  1379. for (i = 0; i < gctx->num_funcs; i++) {
  1380. func_bytes = bsize * gctx->funcs[i].num_blocks;
  1381. @@ -71,10 +71,10 @@
  1382. }
  1383. done:
  1384. + glue_fpu_end(fpu_enabled);
  1385. err = blkcipher_walk_done(desc, walk, nbytes);
  1386. }
  1387. - glue_fpu_end(fpu_enabled);
  1388. return err;
  1389. }
  1390. @@ -194,7 +194,7 @@
  1391. struct scatterlist *src, unsigned int nbytes)
  1392. {
  1393. const unsigned int bsize = 128 / 8;
  1394. - bool fpu_enabled = false;
  1395. + bool fpu_enabled;
  1396. struct blkcipher_walk walk;
  1397. int err;
  1398. @@ -203,12 +203,12 @@
  1399. while ((nbytes = walk.nbytes)) {
  1400. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1401. - desc, fpu_enabled, nbytes);
  1402. + desc, false, nbytes);
  1403. nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
  1404. + glue_fpu_end(fpu_enabled);
  1405. err = blkcipher_walk_done(desc, &walk, nbytes);
  1406. }
  1407. - glue_fpu_end(fpu_enabled);
  1408. return err;
  1409. }
  1410. EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
  1411. @@ -277,7 +277,7 @@
  1412. struct scatterlist *src, unsigned int nbytes)
  1413. {
  1414. const unsigned int bsize = 128 / 8;
  1415. - bool fpu_enabled = false;
  1416. + bool fpu_enabled;
  1417. struct blkcipher_walk walk;
  1418. int err;
  1419. @@ -286,13 +286,12 @@
  1420. while ((nbytes = walk.nbytes) >= bsize) {
  1421. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1422. - desc, fpu_enabled, nbytes);
  1423. + desc, false, nbytes);
  1424. nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
  1425. + glue_fpu_end(fpu_enabled);
  1426. err = blkcipher_walk_done(desc, &walk, nbytes);
  1427. }
  1428. - glue_fpu_end(fpu_enabled);
  1429. -
  1430. if (walk.nbytes) {
  1431. glue_ctr_crypt_final_128bit(
  1432. gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
  1433. @@ -347,7 +346,7 @@
  1434. void *tweak_ctx, void *crypt_ctx)
  1435. {
  1436. const unsigned int bsize = 128 / 8;
  1437. - bool fpu_enabled = false;
  1438. + bool fpu_enabled;
  1439. struct blkcipher_walk walk;
  1440. int err;
  1441. @@ -360,21 +359,21 @@
  1442. /* set minimum length to bsize, for tweak_fn */
  1443. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1444. - desc, fpu_enabled,
  1445. + desc, false,
  1446. nbytes < bsize ? bsize : nbytes);
  1447. -
  1448. /* calculate first value of T */
  1449. tweak_fn(tweak_ctx, walk.iv, walk.iv);
  1450. + glue_fpu_end(fpu_enabled);
  1451. while (nbytes) {
  1452. + fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1453. + desc, false, nbytes);
  1454. nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
  1455. + glue_fpu_end(fpu_enabled);
  1456. err = blkcipher_walk_done(desc, &walk, nbytes);
  1457. nbytes = walk.nbytes;
  1458. }
  1459. -
  1460. - glue_fpu_end(fpu_enabled);
  1461. -
  1462. return err;
  1463. }
  1464. EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
  1465. diff -Nur linux-4.6.6.orig/arch/x86/entry/common.c linux-4.6.6/arch/x86/entry/common.c
  1466. --- linux-4.6.6.orig/arch/x86/entry/common.c 2016-08-10 12:54:57.000000000 +0200
  1467. +++ linux-4.6.6/arch/x86/entry/common.c 2016-08-22 21:27:46.577839498 +0200
  1468. @@ -202,7 +202,7 @@
  1469. #define EXIT_TO_USERMODE_LOOP_FLAGS \
  1470. (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
  1471. - _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY)
  1472. + _TIF_NEED_RESCHED_MASK | _TIF_USER_RETURN_NOTIFY)
  1473. static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
  1474. {
  1475. @@ -218,9 +218,16 @@
  1476. /* We have work to do. */
  1477. local_irq_enable();
  1478. - if (cached_flags & _TIF_NEED_RESCHED)
  1479. + if (cached_flags & _TIF_NEED_RESCHED_MASK)
  1480. schedule();
  1481. +#ifdef ARCH_RT_DELAYS_SIGNAL_SEND
  1482. + if (unlikely(current->forced_info.si_signo)) {
  1483. + struct task_struct *t = current;
  1484. + force_sig_info(t->forced_info.si_signo, &t->forced_info, t);
  1485. + t->forced_info.si_signo = 0;
  1486. + }
  1487. +#endif
  1488. if (cached_flags & _TIF_UPROBE)
  1489. uprobe_notify_resume(regs);
  1490. diff -Nur linux-4.6.6.orig/arch/x86/entry/entry_32.S linux-4.6.6/arch/x86/entry/entry_32.S
  1491. --- linux-4.6.6.orig/arch/x86/entry/entry_32.S 2016-08-10 12:54:57.000000000 +0200
  1492. +++ linux-4.6.6/arch/x86/entry/entry_32.S 2016-08-22 21:27:46.577839498 +0200
  1493. @@ -278,8 +278,24 @@
  1494. ENTRY(resume_kernel)
  1495. DISABLE_INTERRUPTS(CLBR_ANY)
  1496. need_resched:
  1497. + # preempt count == 0 + NEED_RS set?
  1498. cmpl $0, PER_CPU_VAR(__preempt_count)
  1499. +#ifndef CONFIG_PREEMPT_LAZY
  1500. jnz restore_all
  1501. +#else
  1502. + jz test_int_off
  1503. +
  1504. + # atleast preempt count == 0 ?
  1505. + cmpl $_PREEMPT_ENABLED,PER_CPU_VAR(__preempt_count)
  1506. + jne restore_all
  1507. +
  1508. + cmpl $0,TI_preempt_lazy_count(%ebp) # non-zero preempt_lazy_count ?
  1509. + jnz restore_all
  1510. +
  1511. + testl $_TIF_NEED_RESCHED_LAZY, TI_flags(%ebp)
  1512. + jz restore_all
  1513. +test_int_off:
  1514. +#endif
  1515. testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
  1516. jz restore_all
  1517. call preempt_schedule_irq
  1518. diff -Nur linux-4.6.6.orig/arch/x86/entry/entry_64.S linux-4.6.6/arch/x86/entry/entry_64.S
  1519. --- linux-4.6.6.orig/arch/x86/entry/entry_64.S 2016-08-10 12:54:57.000000000 +0200
  1520. +++ linux-4.6.6/arch/x86/entry/entry_64.S 2016-08-22 21:27:46.581839759 +0200
  1521. @@ -511,7 +511,23 @@
  1522. bt $9, EFLAGS(%rsp) /* were interrupts off? */
  1523. jnc 1f
  1524. 0: cmpl $0, PER_CPU_VAR(__preempt_count)
  1525. +#ifndef CONFIG_PREEMPT_LAZY
  1526. jnz 1f
  1527. +#else
  1528. + jz do_preempt_schedule_irq
  1529. +
  1530. + # atleast preempt count == 0 ?
  1531. + cmpl $_PREEMPT_ENABLED,PER_CPU_VAR(__preempt_count)
  1532. + jnz 1f
  1533. +
  1534. + GET_THREAD_INFO(%rcx)
  1535. + cmpl $0, TI_preempt_lazy_count(%rcx)
  1536. + jnz 1f
  1537. +
  1538. + bt $TIF_NEED_RESCHED_LAZY,TI_flags(%rcx)
  1539. + jnc 1f
  1540. +do_preempt_schedule_irq:
  1541. +#endif
  1542. call preempt_schedule_irq
  1543. jmp 0b
  1544. 1:
  1545. @@ -799,6 +815,7 @@
  1546. jmp 2b
  1547. .previous
  1548. +#ifndef CONFIG_PREEMPT_RT_FULL
  1549. /* Call softirq on interrupt stack. Interrupts are off. */
  1550. ENTRY(do_softirq_own_stack)
  1551. pushq %rbp
  1552. @@ -811,6 +828,7 @@
  1553. decl PER_CPU_VAR(irq_count)
  1554. ret
  1555. END(do_softirq_own_stack)
  1556. +#endif
  1557. #ifdef CONFIG_XEN
  1558. idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
  1559. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/preempt.h linux-4.6.6/arch/x86/include/asm/preempt.h
  1560. --- linux-4.6.6.orig/arch/x86/include/asm/preempt.h 2016-08-10 12:54:57.000000000 +0200
  1561. +++ linux-4.6.6/arch/x86/include/asm/preempt.h 2016-08-22 21:27:46.585840012 +0200
  1562. @@ -79,17 +79,33 @@
  1563. * a decrement which hits zero means we have no preempt_count and should
  1564. * reschedule.
  1565. */
  1566. -static __always_inline bool __preempt_count_dec_and_test(void)
  1567. +static __always_inline bool ____preempt_count_dec_and_test(void)
  1568. {
  1569. GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e");
  1570. }
  1571. +static __always_inline bool __preempt_count_dec_and_test(void)
  1572. +{
  1573. + if (____preempt_count_dec_and_test())
  1574. + return true;
  1575. +#ifdef CONFIG_PREEMPT_LAZY
  1576. + return test_thread_flag(TIF_NEED_RESCHED_LAZY);
  1577. +#else
  1578. + return false;
  1579. +#endif
  1580. +}
  1581. +
  1582. /*
  1583. * Returns true when we need to resched and can (barring IRQ state).
  1584. */
  1585. static __always_inline bool should_resched(int preempt_offset)
  1586. {
  1587. +#ifdef CONFIG_PREEMPT_LAZY
  1588. + return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset ||
  1589. + test_thread_flag(TIF_NEED_RESCHED_LAZY));
  1590. +#else
  1591. return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
  1592. +#endif
  1593. }
  1594. #ifdef CONFIG_PREEMPT
  1595. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/signal.h linux-4.6.6/arch/x86/include/asm/signal.h
  1596. --- linux-4.6.6.orig/arch/x86/include/asm/signal.h 2016-08-10 12:54:57.000000000 +0200
  1597. +++ linux-4.6.6/arch/x86/include/asm/signal.h 2016-08-22 21:27:46.585840012 +0200
  1598. @@ -23,6 +23,19 @@
  1599. unsigned long sig[_NSIG_WORDS];
  1600. } sigset_t;
  1601. +/*
  1602. + * Because some traps use the IST stack, we must keep preemption
  1603. + * disabled while calling do_trap(), but do_trap() may call
  1604. + * force_sig_info() which will grab the signal spin_locks for the
  1605. + * task, which in PREEMPT_RT_FULL are mutexes. By defining
  1606. + * ARCH_RT_DELAYS_SIGNAL_SEND the force_sig_info() will set
  1607. + * TIF_NOTIFY_RESUME and set up the signal to be sent on exit of the
  1608. + * trap.
  1609. + */
  1610. +#if defined(CONFIG_PREEMPT_RT_FULL)
  1611. +#define ARCH_RT_DELAYS_SIGNAL_SEND
  1612. +#endif
  1613. +
  1614. #ifndef CONFIG_COMPAT
  1615. typedef sigset_t compat_sigset_t;
  1616. #endif
  1617. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/stackprotector.h linux-4.6.6/arch/x86/include/asm/stackprotector.h
  1618. --- linux-4.6.6.orig/arch/x86/include/asm/stackprotector.h 2016-08-10 12:54:57.000000000 +0200
  1619. +++ linux-4.6.6/arch/x86/include/asm/stackprotector.h 2016-08-22 21:27:46.585840012 +0200
  1620. @@ -59,7 +59,7 @@
  1621. */
  1622. static __always_inline void boot_init_stack_canary(void)
  1623. {
  1624. - u64 canary;
  1625. + u64 uninitialized_var(canary);
  1626. u64 tsc;
  1627. #ifdef CONFIG_X86_64
  1628. @@ -70,8 +70,15 @@
  1629. * of randomness. The TSC only matters for very early init,
  1630. * there it already has some randomness on most systems. Later
  1631. * on during the bootup the random pool has true entropy too.
  1632. + *
  1633. + * For preempt-rt we need to weaken the randomness a bit, as
  1634. + * we can't call into the random generator from atomic context
  1635. + * due to locking constraints. We just leave canary
  1636. + * uninitialized and use the TSC based randomness on top of it.
  1637. */
  1638. +#ifndef CONFIG_PREEMPT_RT_FULL
  1639. get_random_bytes(&canary, sizeof(canary));
  1640. +#endif
  1641. tsc = rdtsc();
  1642. canary += tsc + (tsc << 32UL);
  1643. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/thread_info.h linux-4.6.6/arch/x86/include/asm/thread_info.h
  1644. --- linux-4.6.6.orig/arch/x86/include/asm/thread_info.h 2016-08-10 12:54:57.000000000 +0200
  1645. +++ linux-4.6.6/arch/x86/include/asm/thread_info.h 2016-08-22 21:27:46.589840272 +0200
  1646. @@ -58,6 +58,8 @@
  1647. __u32 status; /* thread synchronous flags */
  1648. __u32 cpu; /* current CPU */
  1649. mm_segment_t addr_limit;
  1650. + int preempt_lazy_count; /* 0 => lazy preemptable
  1651. + <0 => BUG */
  1652. unsigned int sig_on_uaccess_error:1;
  1653. unsigned int uaccess_err:1; /* uaccess failed */
  1654. };
  1655. @@ -95,6 +97,7 @@
  1656. #define TIF_SYSCALL_EMU 6 /* syscall emulation active */
  1657. #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
  1658. #define TIF_SECCOMP 8 /* secure computing */
  1659. +#define TIF_NEED_RESCHED_LAZY 9 /* lazy rescheduling necessary */
  1660. #define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */
  1661. #define TIF_UPROBE 12 /* breakpointed or singlestepping */
  1662. #define TIF_NOTSC 16 /* TSC is not accessible in userland */
  1663. @@ -119,6 +122,7 @@
  1664. #define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
  1665. #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
  1666. #define _TIF_SECCOMP (1 << TIF_SECCOMP)
  1667. +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
  1668. #define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)
  1669. #define _TIF_UPROBE (1 << TIF_UPROBE)
  1670. #define _TIF_NOTSC (1 << TIF_NOTSC)
  1671. @@ -155,6 +159,8 @@
  1672. #define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
  1673. #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
  1674. +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
  1675. +
  1676. #define STACK_WARN (THREAD_SIZE/8)
  1677. /*
  1678. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/tlbflush.h linux-4.6.6/arch/x86/include/asm/tlbflush.h
  1679. --- linux-4.6.6.orig/arch/x86/include/asm/tlbflush.h 2016-08-10 12:54:57.000000000 +0200
  1680. +++ linux-4.6.6/arch/x86/include/asm/tlbflush.h 2016-08-22 21:27:46.589840272 +0200
  1681. @@ -135,7 +135,14 @@
  1682. static inline void __native_flush_tlb(void)
  1683. {
  1684. + /*
  1685. + * if current->mm == NULL then we borrow a mm which may change during a
  1686. + * task switch and therefore we must not be preempted while we write CR3
  1687. + * back.
  1688. + */
  1689. + preempt_disable();
  1690. native_write_cr3(native_read_cr3());
  1691. + preempt_enable();
  1692. }
  1693. static inline void __native_flush_tlb_global_irq_disabled(void)
  1694. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.6.6/arch/x86/include/asm/uv/uv_bau.h
  1695. --- linux-4.6.6.orig/arch/x86/include/asm/uv/uv_bau.h 2016-08-10 12:54:57.000000000 +0200
  1696. +++ linux-4.6.6/arch/x86/include/asm/uv/uv_bau.h 2016-08-22 21:27:46.593840525 +0200
  1697. @@ -615,9 +615,9 @@
  1698. cycles_t send_message;
  1699. cycles_t period_end;
  1700. cycles_t period_time;
  1701. - spinlock_t uvhub_lock;
  1702. - spinlock_t queue_lock;
  1703. - spinlock_t disable_lock;
  1704. + raw_spinlock_t uvhub_lock;
  1705. + raw_spinlock_t queue_lock;
  1706. + raw_spinlock_t disable_lock;
  1707. /* tunables */
  1708. int max_concurr;
  1709. int max_concurr_const;
  1710. @@ -776,15 +776,15 @@
  1711. * to be lowered below the current 'v'. atomic_add_unless can only stop
  1712. * on equal.
  1713. */
  1714. -static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
  1715. +static inline int atomic_inc_unless_ge(raw_spinlock_t *lock, atomic_t *v, int u)
  1716. {
  1717. - spin_lock(lock);
  1718. + raw_spin_lock(lock);
  1719. if (atomic_read(v) >= u) {
  1720. - spin_unlock(lock);
  1721. + raw_spin_unlock(lock);
  1722. return 0;
  1723. }
  1724. atomic_inc(v);
  1725. - spin_unlock(lock);
  1726. + raw_spin_unlock(lock);
  1727. return 1;
  1728. }
  1729. diff -Nur linux-4.6.6.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.6.6/arch/x86/include/asm/uv/uv_hub.h
  1730. --- linux-4.6.6.orig/arch/x86/include/asm/uv/uv_hub.h 2016-08-10 12:54:57.000000000 +0200
  1731. +++ linux-4.6.6/arch/x86/include/asm/uv/uv_hub.h 2016-08-22 21:27:46.593840525 +0200
  1732. @@ -492,7 +492,7 @@
  1733. unsigned short nr_online_cpus;
  1734. unsigned short pnode;
  1735. short memory_nid;
  1736. - spinlock_t nmi_lock; /* obsolete, see uv_hub_nmi */
  1737. + raw_spinlock_t nmi_lock; /* obsolete, see uv_hub_nmi */
  1738. unsigned long nmi_count; /* obsolete, see uv_hub_nmi */
  1739. };
  1740. extern struct uv_blade_info *uv_blade_info;
  1741. diff -Nur linux-4.6.6.orig/arch/x86/Kconfig linux-4.6.6/arch/x86/Kconfig
  1742. --- linux-4.6.6.orig/arch/x86/Kconfig 2016-08-10 12:54:57.000000000 +0200
  1743. +++ linux-4.6.6/arch/x86/Kconfig 2016-08-22 21:27:46.549837706 +0200
  1744. @@ -17,6 +17,7 @@
  1745. ### Arch settings
  1746. config X86
  1747. def_bool y
  1748. + select HAVE_PREEMPT_LAZY
  1749. select ACPI_LEGACY_TABLES_LOOKUP if ACPI
  1750. select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI
  1751. select ANON_INODES
  1752. @@ -230,8 +231,11 @@
  1753. def_bool y
  1754. depends on ISA_DMA_API
  1755. +config RWSEM_GENERIC_SPINLOCK
  1756. + def_bool PREEMPT_RT_FULL
  1757. +
  1758. config RWSEM_XCHGADD_ALGORITHM
  1759. - def_bool y
  1760. + def_bool !RWSEM_GENERIC_SPINLOCK && !PREEMPT_RT_FULL
  1761. config GENERIC_CALIBRATE_DELAY
  1762. def_bool y
  1763. @@ -889,7 +893,7 @@
  1764. config MAXSMP
  1765. bool "Enable Maximum number of SMP Processors and NUMA Nodes"
  1766. depends on X86_64 && SMP && DEBUG_KERNEL
  1767. - select CPUMASK_OFFSTACK
  1768. + select CPUMASK_OFFSTACK if !PREEMPT_RT_FULL
  1769. ---help---
  1770. Enable maximum number of CPUS and NUMA Nodes for this architecture.
  1771. If unsure, say N.
  1772. diff -Nur linux-4.6.6.orig/arch/x86/kernel/apic/io_apic.c linux-4.6.6/arch/x86/kernel/apic/io_apic.c
  1773. --- linux-4.6.6.orig/arch/x86/kernel/apic/io_apic.c 2016-08-10 12:54:57.000000000 +0200
  1774. +++ linux-4.6.6/arch/x86/kernel/apic/io_apic.c 2016-08-22 21:27:46.605841298 +0200
  1775. @@ -1711,7 +1711,8 @@
  1776. static inline bool ioapic_irqd_mask(struct irq_data *data)
  1777. {
  1778. /* If we are moving the irq we need to mask it */
  1779. - if (unlikely(irqd_is_setaffinity_pending(data))) {
  1780. + if (unlikely(irqd_is_setaffinity_pending(data) &&
  1781. + !irqd_irq_inprogress(data))) {
  1782. mask_ioapic_irq(data);
  1783. return true;
  1784. }
  1785. diff -Nur linux-4.6.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.6.6/arch/x86/kernel/apic/x2apic_uv_x.c
  1786. --- linux-4.6.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2016-08-10 12:54:57.000000000 +0200
  1787. +++ linux-4.6.6/arch/x86/kernel/apic/x2apic_uv_x.c 2016-08-22 21:27:46.617842064 +0200
  1788. @@ -755,7 +755,7 @@
  1789. uv_set_scir_bits(bits);
  1790. /* enable next timer period */
  1791. - mod_timer_pinned(timer, jiffies + SCIR_CPU_HB_INTERVAL);
  1792. + mod_timer(timer, jiffies + SCIR_CPU_HB_INTERVAL);
  1793. }
  1794. static void uv_heartbeat_enable(int cpu)
  1795. @@ -764,7 +764,7 @@
  1796. struct timer_list *timer = &uv_cpu_hub_info(cpu)->scir.timer;
  1797. uv_set_cpu_scir_bits(cpu, SCIR_CPU_HEARTBEAT|SCIR_CPU_ACTIVITY);
  1798. - setup_timer(timer, uv_heartbeat, cpu);
  1799. + setup_pinned_timer(timer, uv_heartbeat, cpu);
  1800. timer->expires = jiffies + SCIR_CPU_HB_INTERVAL;
  1801. add_timer_on(timer, cpu);
  1802. uv_cpu_hub_info(cpu)->scir.enabled = 1;
  1803. @@ -950,7 +950,7 @@
  1804. uv_blade_info[blade].pnode = pnode;
  1805. uv_blade_info[blade].nr_possible_cpus = 0;
  1806. uv_blade_info[blade].nr_online_cpus = 0;
  1807. - spin_lock_init(&uv_blade_info[blade].nmi_lock);
  1808. + raw_spin_lock_init(&uv_blade_info[blade].nmi_lock);
  1809. min_pnode = min(pnode, min_pnode);
  1810. max_pnode = max(pnode, max_pnode);
  1811. blade++;
  1812. diff -Nur linux-4.6.6.orig/arch/x86/kernel/asm-offsets.c linux-4.6.6/arch/x86/kernel/asm-offsets.c
  1813. --- linux-4.6.6.orig/arch/x86/kernel/asm-offsets.c 2016-08-10 12:54:57.000000000 +0200
  1814. +++ linux-4.6.6/arch/x86/kernel/asm-offsets.c 2016-08-22 21:27:46.617842064 +0200
  1815. @@ -32,6 +32,7 @@
  1816. OFFSET(TI_flags, thread_info, flags);
  1817. OFFSET(TI_status, thread_info, status);
  1818. OFFSET(TI_addr_limit, thread_info, addr_limit);
  1819. + OFFSET(TI_preempt_lazy_count, thread_info, preempt_lazy_count);
  1820. BLANK();
  1821. OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
  1822. @@ -85,4 +86,5 @@
  1823. BLANK();
  1824. DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
  1825. + DEFINE(_PREEMPT_ENABLED, PREEMPT_ENABLED);
  1826. }
  1827. diff -Nur linux-4.6.6.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.6.6/arch/x86/kernel/cpu/mcheck/mce.c
  1828. --- linux-4.6.6.orig/arch/x86/kernel/cpu/mcheck/mce.c 2016-08-10 12:54:57.000000000 +0200
  1829. +++ linux-4.6.6/arch/x86/kernel/cpu/mcheck/mce.c 2016-08-22 21:27:46.629842834 +0200
  1830. @@ -41,6 +41,8 @@
  1831. #include <linux/debugfs.h>
  1832. #include <linux/irq_work.h>
  1833. #include <linux/export.h>
  1834. +#include <linux/jiffies.h>
  1835. +#include <linux/swork.h>
  1836. #include <asm/processor.h>
  1837. #include <asm/traps.h>
  1838. @@ -1240,7 +1242,7 @@
  1839. static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
  1840. static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
  1841. -static DEFINE_PER_CPU(struct timer_list, mce_timer);
  1842. +static DEFINE_PER_CPU(struct hrtimer, mce_timer);
  1843. static unsigned long mce_adjust_timer_default(unsigned long interval)
  1844. {
  1845. @@ -1249,32 +1251,18 @@
  1846. static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
  1847. -static void __restart_timer(struct timer_list *t, unsigned long interval)
  1848. +static enum hrtimer_restart __restart_timer(struct hrtimer *timer, unsigned long interval)
  1849. {
  1850. - unsigned long when = jiffies + interval;
  1851. - unsigned long flags;
  1852. -
  1853. - local_irq_save(flags);
  1854. -
  1855. - if (timer_pending(t)) {
  1856. - if (time_before(when, t->expires))
  1857. - mod_timer_pinned(t, when);
  1858. - } else {
  1859. - t->expires = round_jiffies(when);
  1860. - add_timer_on(t, smp_processor_id());
  1861. - }
  1862. -
  1863. - local_irq_restore(flags);
  1864. + if (!interval)
  1865. + return HRTIMER_NORESTART;
  1866. + hrtimer_forward_now(timer, ns_to_ktime(jiffies_to_nsecs(interval)));
  1867. + return HRTIMER_RESTART;
  1868. }
  1869. -static void mce_timer_fn(unsigned long data)
  1870. +static enum hrtimer_restart mce_timer_fn(struct hrtimer *timer)
  1871. {
  1872. - struct timer_list *t = this_cpu_ptr(&mce_timer);
  1873. - int cpu = smp_processor_id();
  1874. unsigned long iv;
  1875. - WARN_ON(cpu != data);
  1876. -
  1877. iv = __this_cpu_read(mce_next_interval);
  1878. if (mce_available(this_cpu_ptr(&cpu_info))) {
  1879. @@ -1297,7 +1285,7 @@
  1880. done:
  1881. __this_cpu_write(mce_next_interval, iv);
  1882. - __restart_timer(t, iv);
  1883. + return __restart_timer(timer, iv);
  1884. }
  1885. /*
  1886. @@ -1305,7 +1293,7 @@
  1887. */
  1888. void mce_timer_kick(unsigned long interval)
  1889. {
  1890. - struct timer_list *t = this_cpu_ptr(&mce_timer);
  1891. + struct hrtimer *t = this_cpu_ptr(&mce_timer);
  1892. unsigned long iv = __this_cpu_read(mce_next_interval);
  1893. __restart_timer(t, interval);
  1894. @@ -1320,7 +1308,7 @@
  1895. int cpu;
  1896. for_each_online_cpu(cpu)
  1897. - del_timer_sync(&per_cpu(mce_timer, cpu));
  1898. + hrtimer_cancel(&per_cpu(mce_timer, cpu));
  1899. }
  1900. static void mce_do_trigger(struct work_struct *work)
  1901. @@ -1330,6 +1318,56 @@
  1902. static DECLARE_WORK(mce_trigger_work, mce_do_trigger);
  1903. +static void __mce_notify_work(struct swork_event *event)
  1904. +{
  1905. + /* Not more than two messages every minute */
  1906. + static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
  1907. +
  1908. + /* wake processes polling /dev/mcelog */
  1909. + wake_up_interruptible(&mce_chrdev_wait);
  1910. +
  1911. + /*
  1912. + * There is no risk of missing notifications because
  1913. + * work_pending is always cleared before the function is
  1914. + * executed.
  1915. + */
  1916. + if (mce_helper[0] && !work_pending(&mce_trigger_work))
  1917. + schedule_work(&mce_trigger_work);
  1918. +
  1919. + if (__ratelimit(&ratelimit))
  1920. + pr_info(HW_ERR "Machine check events logged\n");
  1921. +}
  1922. +
  1923. +#ifdef CONFIG_PREEMPT_RT_FULL
  1924. +static bool notify_work_ready __read_mostly;
  1925. +static struct swork_event notify_work;
  1926. +
  1927. +static int mce_notify_work_init(void)
  1928. +{
  1929. + int err;
  1930. +
  1931. + err = swork_get();
  1932. + if (err)
  1933. + return err;
  1934. +
  1935. + INIT_SWORK(&notify_work, __mce_notify_work);
  1936. + notify_work_ready = true;
  1937. + return 0;
  1938. +}
  1939. +
  1940. +static void mce_notify_work(void)
  1941. +{
  1942. + if (notify_work_ready)
  1943. + swork_queue(&notify_work);
  1944. +}
  1945. +#else
  1946. +static void mce_notify_work(void)
  1947. +{
  1948. + __mce_notify_work(NULL);
  1949. +}
  1950. +static inline int mce_notify_work_init(void) { return 0; }
  1951. +#endif
  1952. +
  1953. /*
  1954. * Notify the user(s) about new machine check events.
  1955. * Can be called from interrupt context, but not from machine check/NMI
  1956. @@ -1337,19 +1375,8 @@
  1957. */
  1958. int mce_notify_irq(void)
  1959. {
  1960. - /* Not more than two messages every minute */
  1961. - static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
  1962. -
  1963. if (test_and_clear_bit(0, &mce_need_notify)) {
  1964. - /* wake processes polling /dev/mcelog */
  1965. - wake_up_interruptible(&mce_chrdev_wait);
  1966. -
  1967. - if (mce_helper[0])
  1968. - schedule_work(&mce_trigger_work);
  1969. -
  1970. - if (__ratelimit(&ratelimit))
  1971. - pr_info(HW_ERR "Machine check events logged\n");
  1972. -
  1973. + mce_notify_work();
  1974. return 1;
  1975. }
  1976. return 0;
  1977. @@ -1654,7 +1681,7 @@
  1978. }
  1979. }
  1980. -static void mce_start_timer(unsigned int cpu, struct timer_list *t)
  1981. +static void mce_start_timer(unsigned int cpu, struct hrtimer *t)
  1982. {
  1983. unsigned long iv = check_interval * HZ;
  1984. @@ -1663,16 +1690,17 @@
  1985. per_cpu(mce_next_interval, cpu) = iv;
  1986. - t->expires = round_jiffies(jiffies + iv);
  1987. - add_timer_on(t, cpu);
  1988. + hrtimer_start_range_ns(t, ns_to_ktime(jiffies_to_usecs(iv) * 1000ULL),
  1989. + 0, HRTIMER_MODE_REL_PINNED);
  1990. }
  1991. static void __mcheck_cpu_init_timer(void)
  1992. {
  1993. - struct timer_list *t = this_cpu_ptr(&mce_timer);
  1994. + struct hrtimer *t = this_cpu_ptr(&mce_timer);
  1995. unsigned int cpu = smp_processor_id();
  1996. - setup_timer(t, mce_timer_fn, cpu);
  1997. + hrtimer_init(t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  1998. + t->function = mce_timer_fn;
  1999. mce_start_timer(cpu, t);
  2000. }
  2001. @@ -2393,6 +2421,8 @@
  2002. if (!mce_available(raw_cpu_ptr(&cpu_info)))
  2003. return;
  2004. + hrtimer_cancel(this_cpu_ptr(&mce_timer));
  2005. +
  2006. if (!(action & CPU_TASKS_FROZEN))
  2007. cmci_clear();
  2008. @@ -2415,6 +2445,7 @@
  2009. if (b->init)
  2010. wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
  2011. }
  2012. + __mcheck_cpu_init_timer();
  2013. }
  2014. /* Get notified when a cpu comes on/off. Be hotplug friendly. */
  2015. @@ -2422,7 +2453,6 @@
  2016. mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
  2017. {
  2018. unsigned int cpu = (unsigned long)hcpu;
  2019. - struct timer_list *t = &per_cpu(mce_timer, cpu);
  2020. switch (action & ~CPU_TASKS_FROZEN) {
  2021. case CPU_ONLINE:
  2022. @@ -2442,11 +2472,9 @@
  2023. break;
  2024. case CPU_DOWN_PREPARE:
  2025. smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
  2026. - del_timer_sync(t);
  2027. break;
  2028. case CPU_DOWN_FAILED:
  2029. smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
  2030. - mce_start_timer(cpu, t);
  2031. break;
  2032. }
  2033. @@ -2485,6 +2513,10 @@
  2034. goto err_out;
  2035. }
  2036. + err = mce_notify_work_init();
  2037. + if (err)
  2038. + goto err_out;
  2039. +
  2040. if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
  2041. err = -ENOMEM;
  2042. goto err_out;
  2043. diff -Nur linux-4.6.6.orig/arch/x86/kernel/dumpstack_32.c linux-4.6.6/arch/x86/kernel/dumpstack_32.c
  2044. --- linux-4.6.6.orig/arch/x86/kernel/dumpstack_32.c 2016-08-10 12:54:57.000000000 +0200
  2045. +++ linux-4.6.6/arch/x86/kernel/dumpstack_32.c 2016-08-22 21:27:46.633843092 +0200
  2046. @@ -42,7 +42,7 @@
  2047. unsigned long *stack, unsigned long bp,
  2048. const struct stacktrace_ops *ops, void *data)
  2049. {
  2050. - const unsigned cpu = get_cpu();
  2051. + const unsigned cpu = get_cpu_light();
  2052. int graph = 0;
  2053. u32 *prev_esp;
  2054. @@ -86,7 +86,7 @@
  2055. break;
  2056. touch_nmi_watchdog();
  2057. }
  2058. - put_cpu();
  2059. + put_cpu_light();
  2060. }
  2061. EXPORT_SYMBOL(dump_trace);
  2062. diff -Nur linux-4.6.6.orig/arch/x86/kernel/dumpstack_64.c linux-4.6.6/arch/x86/kernel/dumpstack_64.c
  2063. --- linux-4.6.6.orig/arch/x86/kernel/dumpstack_64.c 2016-08-10 12:54:57.000000000 +0200
  2064. +++ linux-4.6.6/arch/x86/kernel/dumpstack_64.c 2016-08-22 21:27:46.633843092 +0200
  2065. @@ -152,7 +152,7 @@
  2066. unsigned long *stack, unsigned long bp,
  2067. const struct stacktrace_ops *ops, void *data)
  2068. {
  2069. - const unsigned cpu = get_cpu();
  2070. + const unsigned cpu = get_cpu_light();
  2071. struct thread_info *tinfo;
  2072. unsigned long *irq_stack = (unsigned long *)per_cpu(irq_stack_ptr, cpu);
  2073. unsigned long dummy;
  2074. @@ -241,7 +241,7 @@
  2075. * This handles the process stack:
  2076. */
  2077. bp = ops->walk_stack(tinfo, stack, bp, ops, data, NULL, &graph);
  2078. - put_cpu();
  2079. + put_cpu_light();
  2080. }
  2081. EXPORT_SYMBOL(dump_trace);
  2082. @@ -255,7 +255,7 @@
  2083. int cpu;
  2084. int i;
  2085. - preempt_disable();
  2086. + migrate_disable();
  2087. cpu = smp_processor_id();
  2088. irq_stack_end = (unsigned long *)(per_cpu(irq_stack_ptr, cpu));
  2089. @@ -291,7 +291,7 @@
  2090. pr_cont(" %016lx", *stack++);
  2091. touch_nmi_watchdog();
  2092. }
  2093. - preempt_enable();
  2094. + migrate_enable();
  2095. pr_cont("\n");
  2096. show_trace_log_lvl(task, regs, sp, bp, log_lvl);
  2097. diff -Nur linux-4.6.6.orig/arch/x86/kernel/irq_32.c linux-4.6.6/arch/x86/kernel/irq_32.c
  2098. --- linux-4.6.6.orig/arch/x86/kernel/irq_32.c 2016-08-10 12:54:57.000000000 +0200
  2099. +++ linux-4.6.6/arch/x86/kernel/irq_32.c 2016-08-22 21:27:46.637843343 +0200
  2100. @@ -128,6 +128,7 @@
  2101. cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu));
  2102. }
  2103. +#ifndef CONFIG_PREEMPT_RT_FULL
  2104. void do_softirq_own_stack(void)
  2105. {
  2106. struct thread_info *curstk;
  2107. @@ -146,6 +147,7 @@
  2108. call_on_stack(__do_softirq, isp);
  2109. }
  2110. +#endif
  2111. bool handle_irq(struct irq_desc *desc, struct pt_regs *regs)
  2112. {
  2113. diff -Nur linux-4.6.6.orig/arch/x86/kernel/process_32.c linux-4.6.6/arch/x86/kernel/process_32.c
  2114. --- linux-4.6.6.orig/arch/x86/kernel/process_32.c 2016-08-10 12:54:57.000000000 +0200
  2115. +++ linux-4.6.6/arch/x86/kernel/process_32.c 2016-08-22 21:27:46.641843602 +0200
  2116. @@ -35,6 +35,7 @@
  2117. #include <linux/uaccess.h>
  2118. #include <linux/io.h>
  2119. #include <linux/kdebug.h>
  2120. +#include <linux/highmem.h>
  2121. #include <asm/pgtable.h>
  2122. #include <asm/ldt.h>
  2123. @@ -210,6 +211,35 @@
  2124. }
  2125. EXPORT_SYMBOL_GPL(start_thread);
  2126. +#ifdef CONFIG_PREEMPT_RT_FULL
  2127. +static void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p)
  2128. +{
  2129. + int i;
  2130. +
  2131. + /*
  2132. + * Clear @prev's kmap_atomic mappings
  2133. + */
  2134. + for (i = 0; i < prev_p->kmap_idx; i++) {
  2135. + int idx = i + KM_TYPE_NR * smp_processor_id();
  2136. + pte_t *ptep = kmap_pte - idx;
  2137. +
  2138. + kpte_clear_flush(ptep, __fix_to_virt(FIX_KMAP_BEGIN + idx));
  2139. + }
  2140. + /*
  2141. + * Restore @next_p's kmap_atomic mappings
  2142. + */
  2143. + for (i = 0; i < next_p->kmap_idx; i++) {
  2144. + int idx = i + KM_TYPE_NR * smp_processor_id();
  2145. +
  2146. + if (!pte_none(next_p->kmap_pte[i]))
  2147. + set_pte(kmap_pte - idx, next_p->kmap_pte[i]);
  2148. + }
  2149. +}
  2150. +#else
  2151. +static inline void
  2152. +switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p) { }
  2153. +#endif
  2154. +
  2155. /*
  2156. * switch_to(x,y) should switch tasks from x to y.
  2157. @@ -286,6 +316,8 @@
  2158. task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
  2159. __switch_to_xtra(prev_p, next_p, tss);
  2160. + switch_kmaps(prev_p, next_p);
  2161. +
  2162. /*
  2163. * Leave lazy mode, flushing any hypercalls made here.
  2164. * This must be done before restoring TLS segments so
  2165. diff -Nur linux-4.6.6.orig/arch/x86/kvm/lapic.c linux-4.6.6/arch/x86/kvm/lapic.c
  2166. --- linux-4.6.6.orig/arch/x86/kvm/lapic.c 2016-08-10 12:54:57.000000000 +0200
  2167. +++ linux-4.6.6/arch/x86/kvm/lapic.c 2016-08-22 21:27:46.649844115 +0200
  2168. @@ -1870,6 +1870,7 @@
  2169. hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
  2170. HRTIMER_MODE_ABS_PINNED);
  2171. apic->lapic_timer.timer.function = apic_timer_fn;
  2172. + apic->lapic_timer.timer.irqsafe = 1;
  2173. /*
  2174. * APIC is created enabled. This will prevent kvm_lapic_set_base from
  2175. diff -Nur linux-4.6.6.orig/arch/x86/kvm/x86.c linux-4.6.6/arch/x86/kvm/x86.c
  2176. --- linux-4.6.6.orig/arch/x86/kvm/x86.c 2016-08-10 12:54:57.000000000 +0200
  2177. +++ linux-4.6.6/arch/x86/kvm/x86.c 2016-08-22 21:27:46.665845141 +0200
  2178. @@ -5855,6 +5855,13 @@
  2179. goto out;
  2180. }
  2181. +#ifdef CONFIG_PREEMPT_RT_FULL
  2182. + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  2183. + printk(KERN_ERR "RT requires X86_FEATURE_CONSTANT_TSC\n");
  2184. + return -EOPNOTSUPP;
  2185. + }
  2186. +#endif
  2187. +
  2188. r = kvm_mmu_module_init();
  2189. if (r)
  2190. goto out_free_percpu;
  2191. diff -Nur linux-4.6.6.orig/arch/x86/mm/highmem_32.c linux-4.6.6/arch/x86/mm/highmem_32.c
  2192. --- linux-4.6.6.orig/arch/x86/mm/highmem_32.c 2016-08-10 12:54:57.000000000 +0200
  2193. +++ linux-4.6.6/arch/x86/mm/highmem_32.c 2016-08-22 21:27:46.673845654 +0200
  2194. @@ -32,10 +32,11 @@
  2195. */
  2196. void *kmap_atomic_prot(struct page *page, pgprot_t prot)
  2197. {
  2198. + pte_t pte = mk_pte(page, prot);
  2199. unsigned long vaddr;
  2200. int idx, type;
  2201. - preempt_disable();
  2202. + preempt_disable_nort();
  2203. pagefault_disable();
  2204. if (!PageHighMem(page))
  2205. @@ -45,7 +46,10 @@
  2206. idx = type + KM_TYPE_NR*smp_processor_id();
  2207. vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
  2208. BUG_ON(!pte_none(*(kmap_pte-idx)));
  2209. - set_pte(kmap_pte-idx, mk_pte(page, prot));
  2210. +#ifdef CONFIG_PREEMPT_RT_FULL
  2211. + current->kmap_pte[type] = pte;
  2212. +#endif
  2213. + set_pte(kmap_pte-idx, pte);
  2214. arch_flush_lazy_mmu_mode();
  2215. return (void *)vaddr;
  2216. @@ -88,6 +92,9 @@
  2217. * is a bad idea also, in case the page changes cacheability
  2218. * attributes or becomes a protected page in a hypervisor.
  2219. */
  2220. +#ifdef CONFIG_PREEMPT_RT_FULL
  2221. + current->kmap_pte[type] = __pte(0);
  2222. +#endif
  2223. kpte_clear_flush(kmap_pte-idx, vaddr);
  2224. kmap_atomic_idx_pop();
  2225. arch_flush_lazy_mmu_mode();
  2226. @@ -100,7 +107,7 @@
  2227. #endif
  2228. pagefault_enable();
  2229. - preempt_enable();
  2230. + preempt_enable_nort();
  2231. }
  2232. EXPORT_SYMBOL(__kunmap_atomic);
  2233. diff -Nur linux-4.6.6.orig/arch/x86/mm/iomap_32.c linux-4.6.6/arch/x86/mm/iomap_32.c
  2234. --- linux-4.6.6.orig/arch/x86/mm/iomap_32.c 2016-08-10 12:54:57.000000000 +0200
  2235. +++ linux-4.6.6/arch/x86/mm/iomap_32.c 2016-08-22 21:27:46.673845654 +0200
  2236. @@ -56,6 +56,7 @@
  2237. void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot)
  2238. {
  2239. + pte_t pte = pfn_pte(pfn, prot);
  2240. unsigned long vaddr;
  2241. int idx, type;
  2242. @@ -65,7 +66,12 @@
  2243. type = kmap_atomic_idx_push();
  2244. idx = type + KM_TYPE_NR * smp_processor_id();
  2245. vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
  2246. - set_pte(kmap_pte - idx, pfn_pte(pfn, prot));
  2247. + WARN_ON(!pte_none(*(kmap_pte - idx)));
  2248. +
  2249. +#ifdef CONFIG_PREEMPT_RT_FULL
  2250. + current->kmap_pte[type] = pte;
  2251. +#endif
  2252. + set_pte(kmap_pte - idx, pte);
  2253. arch_flush_lazy_mmu_mode();
  2254. return (void *)vaddr;
  2255. @@ -113,6 +119,9 @@
  2256. * is a bad idea also, in case the page changes cacheability
  2257. * attributes or becomes a protected page in a hypervisor.
  2258. */
  2259. +#ifdef CONFIG_PREEMPT_RT_FULL
  2260. + current->kmap_pte[type] = __pte(0);
  2261. +#endif
  2262. kpte_clear_flush(kmap_pte-idx, vaddr);
  2263. kmap_atomic_idx_pop();
  2264. }
  2265. diff -Nur linux-4.6.6.orig/arch/x86/platform/uv/tlb_uv.c linux-4.6.6/arch/x86/platform/uv/tlb_uv.c
  2266. --- linux-4.6.6.orig/arch/x86/platform/uv/tlb_uv.c 2016-08-10 12:54:57.000000000 +0200
  2267. +++ linux-4.6.6/arch/x86/platform/uv/tlb_uv.c 2016-08-22 21:27:46.681846167 +0200
  2268. @@ -714,9 +714,9 @@
  2269. quiesce_local_uvhub(hmaster);
  2270. - spin_lock(&hmaster->queue_lock);
  2271. + raw_spin_lock(&hmaster->queue_lock);
  2272. reset_with_ipi(&bau_desc->distribution, bcp);
  2273. - spin_unlock(&hmaster->queue_lock);
  2274. + raw_spin_unlock(&hmaster->queue_lock);
  2275. end_uvhub_quiesce(hmaster);
  2276. @@ -736,9 +736,9 @@
  2277. quiesce_local_uvhub(hmaster);
  2278. - spin_lock(&hmaster->queue_lock);
  2279. + raw_spin_lock(&hmaster->queue_lock);
  2280. reset_with_ipi(&bau_desc->distribution, bcp);
  2281. - spin_unlock(&hmaster->queue_lock);
  2282. + raw_spin_unlock(&hmaster->queue_lock);
  2283. end_uvhub_quiesce(hmaster);
  2284. @@ -759,7 +759,7 @@
  2285. cycles_t tm1;
  2286. hmaster = bcp->uvhub_master;
  2287. - spin_lock(&hmaster->disable_lock);
  2288. + raw_spin_lock(&hmaster->disable_lock);
  2289. if (!bcp->baudisabled) {
  2290. stat->s_bau_disabled++;
  2291. tm1 = get_cycles();
  2292. @@ -772,7 +772,7 @@
  2293. }
  2294. }
  2295. }
  2296. - spin_unlock(&hmaster->disable_lock);
  2297. + raw_spin_unlock(&hmaster->disable_lock);
  2298. }
  2299. static void count_max_concurr(int stat, struct bau_control *bcp,
  2300. @@ -835,7 +835,7 @@
  2301. */
  2302. static void uv1_throttle(struct bau_control *hmaster, struct ptc_stats *stat)
  2303. {
  2304. - spinlock_t *lock = &hmaster->uvhub_lock;
  2305. + raw_spinlock_t *lock = &hmaster->uvhub_lock;
  2306. atomic_t *v;
  2307. v = &hmaster->active_descriptor_count;
  2308. @@ -968,7 +968,7 @@
  2309. struct bau_control *hmaster;
  2310. hmaster = bcp->uvhub_master;
  2311. - spin_lock(&hmaster->disable_lock);
  2312. + raw_spin_lock(&hmaster->disable_lock);
  2313. if (bcp->baudisabled && (get_cycles() >= bcp->set_bau_on_time)) {
  2314. stat->s_bau_reenabled++;
  2315. for_each_present_cpu(tcpu) {
  2316. @@ -980,10 +980,10 @@
  2317. tbcp->period_giveups = 0;
  2318. }
  2319. }
  2320. - spin_unlock(&hmaster->disable_lock);
  2321. + raw_spin_unlock(&hmaster->disable_lock);
  2322. return 0;
  2323. }
  2324. - spin_unlock(&hmaster->disable_lock);
  2325. + raw_spin_unlock(&hmaster->disable_lock);
  2326. return -1;
  2327. }
  2328. @@ -1901,9 +1901,9 @@
  2329. bcp->cong_reps = congested_reps;
  2330. bcp->disabled_period = sec_2_cycles(disabled_period);
  2331. bcp->giveup_limit = giveup_limit;
  2332. - spin_lock_init(&bcp->queue_lock);
  2333. - spin_lock_init(&bcp->uvhub_lock);
  2334. - spin_lock_init(&bcp->disable_lock);
  2335. + raw_spin_lock_init(&bcp->queue_lock);
  2336. + raw_spin_lock_init(&bcp->uvhub_lock);
  2337. + raw_spin_lock_init(&bcp->disable_lock);
  2338. }
  2339. }
  2340. diff -Nur linux-4.6.6.orig/arch/x86/platform/uv/uv_time.c linux-4.6.6/arch/x86/platform/uv/uv_time.c
  2341. --- linux-4.6.6.orig/arch/x86/platform/uv/uv_time.c 2016-08-10 12:54:57.000000000 +0200
  2342. +++ linux-4.6.6/arch/x86/platform/uv/uv_time.c 2016-08-22 21:27:46.681846167 +0200
  2343. @@ -57,7 +57,7 @@
  2344. /* There is one of these allocated per node */
  2345. struct uv_rtc_timer_head {
  2346. - spinlock_t lock;
  2347. + raw_spinlock_t lock;
  2348. /* next cpu waiting for timer, local node relative: */
  2349. int next_cpu;
  2350. /* number of cpus on this node: */
  2351. @@ -177,7 +177,7 @@
  2352. uv_rtc_deallocate_timers();
  2353. return -ENOMEM;
  2354. }
  2355. - spin_lock_init(&head->lock);
  2356. + raw_spin_lock_init(&head->lock);
  2357. head->ncpus = uv_blade_nr_possible_cpus(bid);
  2358. head->next_cpu = -1;
  2359. blade_info[bid] = head;
  2360. @@ -231,7 +231,7 @@
  2361. unsigned long flags;
  2362. int next_cpu;
  2363. - spin_lock_irqsave(&head->lock, flags);
  2364. + raw_spin_lock_irqsave(&head->lock, flags);
  2365. next_cpu = head->next_cpu;
  2366. *t = expires;
  2367. @@ -243,12 +243,12 @@
  2368. if (uv_setup_intr(cpu, expires)) {
  2369. *t = ULLONG_MAX;
  2370. uv_rtc_find_next_timer(head, pnode);
  2371. - spin_unlock_irqrestore(&head->lock, flags);
  2372. + raw_spin_unlock_irqrestore(&head->lock, flags);
  2373. return -ETIME;
  2374. }
  2375. }
  2376. - spin_unlock_irqrestore(&head->lock, flags);
  2377. + raw_spin_unlock_irqrestore(&head->lock, flags);
  2378. return 0;
  2379. }
  2380. @@ -267,7 +267,7 @@
  2381. unsigned long flags;
  2382. int rc = 0;
  2383. - spin_lock_irqsave(&head->lock, flags);
  2384. + raw_spin_lock_irqsave(&head->lock, flags);
  2385. if ((head->next_cpu == bcpu && uv_read_rtc(NULL) >= *t) || force)
  2386. rc = 1;
  2387. @@ -279,7 +279,7 @@
  2388. uv_rtc_find_next_timer(head, pnode);
  2389. }
  2390. - spin_unlock_irqrestore(&head->lock, flags);
  2391. + raw_spin_unlock_irqrestore(&head->lock, flags);
  2392. return rc;
  2393. }
  2394. @@ -299,13 +299,18 @@
  2395. static cycle_t uv_read_rtc(struct clocksource *cs)
  2396. {
  2397. unsigned long offset;
  2398. + cycle_t cycles;
  2399. + preempt_disable();
  2400. if (uv_get_min_hub_revision_id() == 1)
  2401. offset = 0;
  2402. else
  2403. offset = (uv_blade_processor_id() * L1_CACHE_BYTES) % PAGE_SIZE;
  2404. - return (cycle_t)uv_read_local_mmr(UVH_RTC | offset);
  2405. + cycles = (cycle_t)uv_read_local_mmr(UVH_RTC | offset);
  2406. + preempt_enable();
  2407. +
  2408. + return cycles;
  2409. }
  2410. /*
  2411. diff -Nur linux-4.6.6.orig/block/blk-core.c linux-4.6.6/block/blk-core.c
  2412. --- linux-4.6.6.orig/block/blk-core.c 2016-08-10 12:54:57.000000000 +0200
  2413. +++ linux-4.6.6/block/blk-core.c 2016-08-22 21:27:46.693846941 +0200
  2414. @@ -125,6 +125,9 @@
  2415. INIT_LIST_HEAD(&rq->queuelist);
  2416. INIT_LIST_HEAD(&rq->timeout_list);
  2417. +#ifdef CONFIG_PREEMPT_RT_FULL
  2418. + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
  2419. +#endif
  2420. rq->cpu = -1;
  2421. rq->q = q;
  2422. rq->__sector = (sector_t) -1;
  2423. @@ -233,7 +236,7 @@
  2424. **/
  2425. void blk_start_queue(struct request_queue *q)
  2426. {
  2427. - WARN_ON(!irqs_disabled());
  2428. + WARN_ON_NONRT(!irqs_disabled());
  2429. queue_flag_clear(QUEUE_FLAG_STOPPED, q);
  2430. __blk_run_queue(q);
  2431. @@ -657,7 +660,7 @@
  2432. if (nowait)
  2433. return -EBUSY;
  2434. - ret = wait_event_interruptible(q->mq_freeze_wq,
  2435. + ret = swait_event_interruptible(q->mq_freeze_wq,
  2436. !atomic_read(&q->mq_freeze_depth) ||
  2437. blk_queue_dying(q));
  2438. if (blk_queue_dying(q))
  2439. @@ -677,7 +680,7 @@
  2440. struct request_queue *q =
  2441. container_of(ref, struct request_queue, q_usage_counter);
  2442. - wake_up_all(&q->mq_freeze_wq);
  2443. + swake_up_all(&q->mq_freeze_wq);
  2444. }
  2445. static void blk_rq_timed_out_timer(unsigned long data)
  2446. @@ -746,7 +749,7 @@
  2447. q->bypass_depth = 1;
  2448. __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
  2449. - init_waitqueue_head(&q->mq_freeze_wq);
  2450. + init_swait_queue_head(&q->mq_freeze_wq);
  2451. /*
  2452. * Init percpu_ref in atomic mode so that it's faster to shutdown.
  2453. @@ -3209,7 +3212,7 @@
  2454. blk_run_queue_async(q);
  2455. else
  2456. __blk_run_queue(q);
  2457. - spin_unlock(q->queue_lock);
  2458. + spin_unlock_irq(q->queue_lock);
  2459. }
  2460. static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
  2461. @@ -3257,7 +3260,6 @@
  2462. void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
  2463. {
  2464. struct request_queue *q;
  2465. - unsigned long flags;
  2466. struct request *rq;
  2467. LIST_HEAD(list);
  2468. unsigned int depth;
  2469. @@ -3277,11 +3279,6 @@
  2470. q = NULL;
  2471. depth = 0;
  2472. - /*
  2473. - * Save and disable interrupts here, to avoid doing it for every
  2474. - * queue lock we have to take.
  2475. - */
  2476. - local_irq_save(flags);
  2477. while (!list_empty(&list)) {
  2478. rq = list_entry_rq(list.next);
  2479. list_del_init(&rq->queuelist);
  2480. @@ -3294,7 +3291,7 @@
  2481. queue_unplugged(q, depth, from_schedule);
  2482. q = rq->q;
  2483. depth = 0;
  2484. - spin_lock(q->queue_lock);
  2485. + spin_lock_irq(q->queue_lock);
  2486. }
  2487. /*
  2488. @@ -3321,8 +3318,6 @@
  2489. */
  2490. if (q)
  2491. queue_unplugged(q, depth, from_schedule);
  2492. -
  2493. - local_irq_restore(flags);
  2494. }
  2495. void blk_finish_plug(struct blk_plug *plug)
  2496. diff -Nur linux-4.6.6.orig/block/blk-ioc.c linux-4.6.6/block/blk-ioc.c
  2497. --- linux-4.6.6.orig/block/blk-ioc.c 2016-08-10 12:54:57.000000000 +0200
  2498. +++ linux-4.6.6/block/blk-ioc.c 2016-08-22 21:27:46.697847194 +0200
  2499. @@ -7,6 +7,7 @@
  2500. #include <linux/bio.h>
  2501. #include <linux/blkdev.h>
  2502. #include <linux/slab.h>
  2503. +#include <linux/delay.h>
  2504. #include "blk.h"
  2505. @@ -109,7 +110,7 @@
  2506. spin_unlock(q->queue_lock);
  2507. } else {
  2508. spin_unlock_irqrestore(&ioc->lock, flags);
  2509. - cpu_relax();
  2510. + cpu_chill();
  2511. spin_lock_irqsave_nested(&ioc->lock, flags, 1);
  2512. }
  2513. }
  2514. @@ -187,7 +188,7 @@
  2515. spin_unlock(icq->q->queue_lock);
  2516. } else {
  2517. spin_unlock_irqrestore(&ioc->lock, flags);
  2518. - cpu_relax();
  2519. + cpu_chill();
  2520. goto retry;
  2521. }
  2522. }
  2523. diff -Nur linux-4.6.6.orig/block/blk-mq.c linux-4.6.6/block/blk-mq.c
  2524. --- linux-4.6.6.orig/block/blk-mq.c 2016-08-10 12:54:57.000000000 +0200
  2525. +++ linux-4.6.6/block/blk-mq.c 2016-08-22 21:27:46.705847707 +0200
  2526. @@ -92,7 +92,7 @@
  2527. static void blk_mq_freeze_queue_wait(struct request_queue *q)
  2528. {
  2529. - wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
  2530. + swait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
  2531. }
  2532. /*
  2533. @@ -130,7 +130,7 @@
  2534. WARN_ON_ONCE(freeze_depth < 0);
  2535. if (!freeze_depth) {
  2536. percpu_ref_reinit(&q->q_usage_counter);
  2537. - wake_up_all(&q->mq_freeze_wq);
  2538. + swake_up_all(&q->mq_freeze_wq);
  2539. }
  2540. }
  2541. EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
  2542. @@ -149,7 +149,7 @@
  2543. * dying, we need to ensure that processes currently waiting on
  2544. * the queue are notified as well.
  2545. */
  2546. - wake_up_all(&q->mq_freeze_wq);
  2547. + swake_up_all(&q->mq_freeze_wq);
  2548. }
  2549. bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
  2550. @@ -196,6 +196,9 @@
  2551. rq->resid_len = 0;
  2552. rq->sense = NULL;
  2553. +#ifdef CONFIG_PREEMPT_RT_FULL
  2554. + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
  2555. +#endif
  2556. INIT_LIST_HEAD(&rq->timeout_list);
  2557. rq->timeout = 0;
  2558. @@ -323,6 +326,17 @@
  2559. }
  2560. EXPORT_SYMBOL(blk_mq_end_request);
  2561. +#ifdef CONFIG_PREEMPT_RT_FULL
  2562. +
  2563. +void __blk_mq_complete_request_remote_work(struct work_struct *work)
  2564. +{
  2565. + struct request *rq = container_of(work, struct request, work);
  2566. +
  2567. + rq->q->softirq_done_fn(rq);
  2568. +}
  2569. +
  2570. +#else
  2571. +
  2572. static void __blk_mq_complete_request_remote(void *data)
  2573. {
  2574. struct request *rq = data;
  2575. @@ -330,6 +344,8 @@
  2576. rq->q->softirq_done_fn(rq);
  2577. }
  2578. +#endif
  2579. +
  2580. static void blk_mq_ipi_complete_request(struct request *rq)
  2581. {
  2582. struct blk_mq_ctx *ctx = rq->mq_ctx;
  2583. @@ -341,19 +357,23 @@
  2584. return;
  2585. }
  2586. - cpu = get_cpu();
  2587. + cpu = get_cpu_light();
  2588. if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
  2589. shared = cpus_share_cache(cpu, ctx->cpu);
  2590. if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
  2591. +#ifdef CONFIG_PREEMPT_RT_FULL
  2592. + schedule_work_on(ctx->cpu, &rq->work);
  2593. +#else
  2594. rq->csd.func = __blk_mq_complete_request_remote;
  2595. rq->csd.info = rq;
  2596. rq->csd.flags = 0;
  2597. smp_call_function_single_async(ctx->cpu, &rq->csd);
  2598. +#endif
  2599. } else {
  2600. rq->q->softirq_done_fn(rq);
  2601. }
  2602. - put_cpu();
  2603. + put_cpu_light();
  2604. }
  2605. static void __blk_mq_complete_request(struct request *rq)
  2606. @@ -868,14 +888,14 @@
  2607. return;
  2608. if (!async) {
  2609. - int cpu = get_cpu();
  2610. + int cpu = get_cpu_light();
  2611. if (cpumask_test_cpu(cpu, hctx->cpumask)) {
  2612. __blk_mq_run_hw_queue(hctx);
  2613. - put_cpu();
  2614. + put_cpu_light();
  2615. return;
  2616. }
  2617. - put_cpu();
  2618. + put_cpu_light();
  2619. }
  2620. kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
  2621. @@ -1621,7 +1641,7 @@
  2622. {
  2623. struct blk_mq_hw_ctx *hctx = data;
  2624. - if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
  2625. + if (action == CPU_POST_DEAD)
  2626. return blk_mq_hctx_cpu_offline(hctx, cpu);
  2627. /*
  2628. diff -Nur linux-4.6.6.orig/block/blk-mq-cpu.c linux-4.6.6/block/blk-mq-cpu.c
  2629. --- linux-4.6.6.orig/block/blk-mq-cpu.c 2016-08-10 12:54:57.000000000 +0200
  2630. +++ linux-4.6.6/block/blk-mq-cpu.c 2016-08-22 21:27:46.701847454 +0200
  2631. @@ -16,7 +16,7 @@
  2632. #include "blk-mq.h"
  2633. static LIST_HEAD(blk_mq_cpu_notify_list);
  2634. -static DEFINE_RAW_SPINLOCK(blk_mq_cpu_notify_lock);
  2635. +static DEFINE_SPINLOCK(blk_mq_cpu_notify_lock);
  2636. static int blk_mq_main_cpu_notify(struct notifier_block *self,
  2637. unsigned long action, void *hcpu)
  2638. @@ -25,7 +25,10 @@
  2639. struct blk_mq_cpu_notifier *notify;
  2640. int ret = NOTIFY_OK;
  2641. - raw_spin_lock(&blk_mq_cpu_notify_lock);
  2642. + if (action != CPU_POST_DEAD)
  2643. + return NOTIFY_OK;
  2644. +
  2645. + spin_lock(&blk_mq_cpu_notify_lock);
  2646. list_for_each_entry(notify, &blk_mq_cpu_notify_list, list) {
  2647. ret = notify->notify(notify->data, action, cpu);
  2648. @@ -33,7 +36,7 @@
  2649. break;
  2650. }
  2651. - raw_spin_unlock(&blk_mq_cpu_notify_lock);
  2652. + spin_unlock(&blk_mq_cpu_notify_lock);
  2653. return ret;
  2654. }
  2655. @@ -41,16 +44,16 @@
  2656. {
  2657. BUG_ON(!notifier->notify);
  2658. - raw_spin_lock(&blk_mq_cpu_notify_lock);
  2659. + spin_lock(&blk_mq_cpu_notify_lock);
  2660. list_add_tail(&notifier->list, &blk_mq_cpu_notify_list);
  2661. - raw_spin_unlock(&blk_mq_cpu_notify_lock);
  2662. + spin_unlock(&blk_mq_cpu_notify_lock);
  2663. }
  2664. void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier)
  2665. {
  2666. - raw_spin_lock(&blk_mq_cpu_notify_lock);
  2667. + spin_lock(&blk_mq_cpu_notify_lock);
  2668. list_del(&notifier->list);
  2669. - raw_spin_unlock(&blk_mq_cpu_notify_lock);
  2670. + spin_unlock(&blk_mq_cpu_notify_lock);
  2671. }
  2672. void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
  2673. diff -Nur linux-4.6.6.orig/block/blk-mq.h linux-4.6.6/block/blk-mq.h
  2674. --- linux-4.6.6.orig/block/blk-mq.h 2016-08-10 12:54:57.000000000 +0200
  2675. +++ linux-4.6.6/block/blk-mq.h 2016-08-22 21:27:46.705847707 +0200
  2676. @@ -86,12 +86,12 @@
  2677. */
  2678. static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
  2679. {
  2680. - return __blk_mq_get_ctx(q, get_cpu());
  2681. + return __blk_mq_get_ctx(q, get_cpu_light());
  2682. }
  2683. static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
  2684. {
  2685. - put_cpu();
  2686. + put_cpu_light();
  2687. }
  2688. struct blk_mq_alloc_data {
  2689. diff -Nur linux-4.6.6.orig/block/blk-softirq.c linux-4.6.6/block/blk-softirq.c
  2690. --- linux-4.6.6.orig/block/blk-softirq.c 2016-08-10 12:54:57.000000000 +0200
  2691. +++ linux-4.6.6/block/blk-softirq.c 2016-08-22 21:27:46.705847707 +0200
  2692. @@ -51,6 +51,7 @@
  2693. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  2694. local_irq_restore(flags);
  2695. + preempt_check_resched_rt();
  2696. }
  2697. /*
  2698. @@ -93,6 +94,7 @@
  2699. this_cpu_ptr(&blk_cpu_done));
  2700. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  2701. local_irq_enable();
  2702. + preempt_check_resched_rt();
  2703. }
  2704. return NOTIFY_OK;
  2705. @@ -150,6 +152,7 @@
  2706. goto do_local;
  2707. local_irq_restore(flags);
  2708. + preempt_check_resched_rt();
  2709. }
  2710. /**
  2711. diff -Nur linux-4.6.6.orig/block/bounce.c linux-4.6.6/block/bounce.c
  2712. --- linux-4.6.6.orig/block/bounce.c 2016-08-10 12:54:57.000000000 +0200
  2713. +++ linux-4.6.6/block/bounce.c 2016-08-22 21:27:46.705847707 +0200
  2714. @@ -55,11 +55,11 @@
  2715. unsigned long flags;
  2716. unsigned char *vto;
  2717. - local_irq_save(flags);
  2718. + local_irq_save_nort(flags);
  2719. vto = kmap_atomic(to->bv_page);
  2720. memcpy(vto + to->bv_offset, vfrom, to->bv_len);
  2721. kunmap_atomic(vto);
  2722. - local_irq_restore(flags);
  2723. + local_irq_restore_nort(flags);
  2724. }
  2725. #else /* CONFIG_HIGHMEM */
  2726. diff -Nur linux-4.6.6.orig/block/genhd.c linux-4.6.6/block/genhd.c
  2727. --- linux-4.6.6.orig/block/genhd.c 2016-08-10 12:54:57.000000000 +0200
  2728. +++ linux-4.6.6/block/genhd.c 2016-08-22 21:27:46.709847967 +0200
  2729. @@ -1523,12 +1523,7 @@
  2730. if (--ev->block)
  2731. goto out_unlock;
  2732. - /*
  2733. - * Not exactly a latency critical operation, set poll timer
  2734. - * slack to 25% and kick event check.
  2735. - */
  2736. intv = disk_events_poll_jiffies(disk);
  2737. - set_timer_slack(&ev->dwork.timer, intv / 4);
  2738. if (check_now)
  2739. queue_delayed_work(system_freezable_power_efficient_wq,
  2740. &ev->dwork, 0);
  2741. diff -Nur linux-4.6.6.orig/crypto/algapi.c linux-4.6.6/crypto/algapi.c
  2742. --- linux-4.6.6.orig/crypto/algapi.c 2016-08-10 12:54:57.000000000 +0200
  2743. +++ linux-4.6.6/crypto/algapi.c 2016-08-22 21:27:46.721848733 +0200
  2744. @@ -718,13 +718,13 @@
  2745. int crypto_register_notifier(struct notifier_block *nb)
  2746. {
  2747. - return blocking_notifier_chain_register(&crypto_chain, nb);
  2748. + return srcu_notifier_chain_register(&crypto_chain, nb);
  2749. }
  2750. EXPORT_SYMBOL_GPL(crypto_register_notifier);
  2751. int crypto_unregister_notifier(struct notifier_block *nb)
  2752. {
  2753. - return blocking_notifier_chain_unregister(&crypto_chain, nb);
  2754. + return srcu_notifier_chain_unregister(&crypto_chain, nb);
  2755. }
  2756. EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
  2757. diff -Nur linux-4.6.6.orig/crypto/api.c linux-4.6.6/crypto/api.c
  2758. --- linux-4.6.6.orig/crypto/api.c 2016-08-10 12:54:57.000000000 +0200
  2759. +++ linux-4.6.6/crypto/api.c 2016-08-22 21:27:46.721848733 +0200
  2760. @@ -31,7 +31,7 @@
  2761. DECLARE_RWSEM(crypto_alg_sem);
  2762. EXPORT_SYMBOL_GPL(crypto_alg_sem);
  2763. -BLOCKING_NOTIFIER_HEAD(crypto_chain);
  2764. +SRCU_NOTIFIER_HEAD(crypto_chain);
  2765. EXPORT_SYMBOL_GPL(crypto_chain);
  2766. static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg);
  2767. @@ -236,10 +236,10 @@
  2768. {
  2769. int ok;
  2770. - ok = blocking_notifier_call_chain(&crypto_chain, val, v);
  2771. + ok = srcu_notifier_call_chain(&crypto_chain, val, v);
  2772. if (ok == NOTIFY_DONE) {
  2773. request_module("cryptomgr");
  2774. - ok = blocking_notifier_call_chain(&crypto_chain, val, v);
  2775. + ok = srcu_notifier_call_chain(&crypto_chain, val, v);
  2776. }
  2777. return ok;
  2778. diff -Nur linux-4.6.6.orig/crypto/internal.h linux-4.6.6/crypto/internal.h
  2779. --- linux-4.6.6.orig/crypto/internal.h 2016-08-10 12:54:57.000000000 +0200
  2780. +++ linux-4.6.6/crypto/internal.h 2016-08-22 21:27:46.725848993 +0200
  2781. @@ -47,7 +47,7 @@
  2782. extern struct list_head crypto_alg_list;
  2783. extern struct rw_semaphore crypto_alg_sem;
  2784. -extern struct blocking_notifier_head crypto_chain;
  2785. +extern struct srcu_notifier_head crypto_chain;
  2786. #ifdef CONFIG_PROC_FS
  2787. void __init crypto_init_proc(void);
  2788. @@ -146,7 +146,7 @@
  2789. static inline void crypto_notify(unsigned long val, void *v)
  2790. {
  2791. - blocking_notifier_call_chain(&crypto_chain, val, v);
  2792. + srcu_notifier_call_chain(&crypto_chain, val, v);
  2793. }
  2794. #endif /* _CRYPTO_INTERNAL_H */
  2795. diff -Nur linux-4.6.6.orig/Documentation/hwlat_detector.txt linux-4.6.6/Documentation/hwlat_detector.txt
  2796. --- linux-4.6.6.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100
  2797. +++ linux-4.6.6/Documentation/hwlat_detector.txt 2016-08-22 21:27:46.213816153 +0200
  2798. @@ -0,0 +1,64 @@
  2799. +Introduction:
  2800. +-------------
  2801. +
  2802. +The module hwlat_detector is a special purpose kernel module that is used to
  2803. +detect large system latencies induced by the behavior of certain underlying
  2804. +hardware or firmware, independent of Linux itself. The code was developed
  2805. +originally to detect SMIs (System Management Interrupts) on x86 systems,
  2806. +however there is nothing x86 specific about this patchset. It was
  2807. +originally written for use by the "RT" patch since the Real Time
  2808. +kernel is highly latency sensitive.
  2809. +
  2810. +SMIs are usually not serviced by the Linux kernel, which typically does not
  2811. +even know that they are occuring. SMIs are instead are set up by BIOS code
  2812. +and are serviced by BIOS code, usually for "critical" events such as
  2813. +management of thermal sensors and fans. Sometimes though, SMIs are used for
  2814. +other tasks and those tasks can spend an inordinate amount of time in the
  2815. +handler (sometimes measured in milliseconds). Obviously this is a problem if
  2816. +you are trying to keep event service latencies down in the microsecond range.
  2817. +
  2818. +The hardware latency detector works by hogging all of the cpus for configurable
  2819. +amounts of time (by calling stop_machine()), polling the CPU Time Stamp Counter
  2820. +for some period, then looking for gaps in the TSC data. Any gap indicates a
  2821. +time when the polling was interrupted and since the machine is stopped and
  2822. +interrupts turned off the only thing that could do that would be an SMI.
  2823. +
  2824. +Note that the SMI detector should *NEVER* be used in a production environment.
  2825. +It is intended to be run manually to determine if the hardware platform has a
  2826. +problem with long system firmware service routines.
  2827. +
  2828. +Usage:
  2829. +------
  2830. +
  2831. +Loading the module hwlat_detector passing the parameter "enabled=1" (or by
  2832. +setting the "enable" entry in "hwlat_detector" debugfs toggled on) is the only
  2833. +step required to start the hwlat_detector. It is possible to redefine the
  2834. +threshold in microseconds (us) above which latency spikes will be taken
  2835. +into account (parameter "threshold=").
  2836. +
  2837. +Example:
  2838. +
  2839. + # modprobe hwlat_detector enabled=1 threshold=100
  2840. +
  2841. +After the module is loaded, it creates a directory named "hwlat_detector" under
  2842. +the debugfs mountpoint, "/debug/hwlat_detector" for this text. It is necessary
  2843. +to have debugfs mounted, which might be on /sys/debug on your system.
  2844. +
  2845. +The /debug/hwlat_detector interface contains the following files:
  2846. +
  2847. +count - number of latency spikes observed since last reset
  2848. +enable - a global enable/disable toggle (0/1), resets count
  2849. +max - maximum hardware latency actually observed (usecs)
  2850. +sample - a pipe from which to read current raw sample data
  2851. + in the format <timestamp> <latency observed usecs>
  2852. + (can be opened O_NONBLOCK for a single sample)
  2853. +threshold - minimum latency value to be considered (usecs)
  2854. +width - time period to sample with CPUs held (usecs)
  2855. + must be less than the total window size (enforced)
  2856. +window - total period of sampling, width being inside (usecs)
  2857. +
  2858. +By default we will set width to 500,000 and window to 1,000,000, meaning that
  2859. +we will sample every 1,000,000 usecs (1s) for 500,000 usecs (0.5s). If we
  2860. +observe any latencies that exceed the threshold (initially 100 usecs),
  2861. +then we write to a global sample ring buffer of 8K samples, which is
  2862. +consumed by reading from the "sample" (pipe) debugfs file interface.
  2863. diff -Nur linux-4.6.6.orig/Documentation/sysrq.txt linux-4.6.6/Documentation/sysrq.txt
  2864. --- linux-4.6.6.orig/Documentation/sysrq.txt 2016-08-10 12:54:57.000000000 +0200
  2865. +++ linux-4.6.6/Documentation/sysrq.txt 2016-08-22 21:27:46.225816925 +0200
  2866. @@ -59,10 +59,17 @@
  2867. On other - If you know of the key combos for other architectures, please
  2868. let me know so I can add them to this section.
  2869. -On all - write a character to /proc/sysrq-trigger. e.g.:
  2870. -
  2871. +On all - write a character to /proc/sysrq-trigger, e.g.:
  2872. echo t > /proc/sysrq-trigger
  2873. +On all - Enable network SysRq by writing a cookie to icmp_echo_sysrq, e.g.
  2874. + echo 0x01020304 >/proc/sys/net/ipv4/icmp_echo_sysrq
  2875. + Send an ICMP echo request with this pattern plus the particular
  2876. + SysRq command key. Example:
  2877. + # ping -c1 -s57 -p0102030468
  2878. + will trigger the SysRq-H (help) command.
  2879. +
  2880. +
  2881. * What are the 'command' keys?
  2882. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  2883. 'b' - Will immediately reboot the system without syncing or unmounting
  2884. diff -Nur linux-4.6.6.orig/Documentation/trace/histograms.txt linux-4.6.6/Documentation/trace/histograms.txt
  2885. --- linux-4.6.6.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100
  2886. +++ linux-4.6.6/Documentation/trace/histograms.txt 2016-08-22 21:27:46.229817185 +0200
  2887. @@ -0,0 +1,186 @@
  2888. + Using the Linux Kernel Latency Histograms
  2889. +
  2890. +
  2891. +This document gives a short explanation how to enable, configure and use
  2892. +latency histograms. Latency histograms are primarily relevant in the
  2893. +context of real-time enabled kernels (CONFIG_PREEMPT/CONFIG_PREEMPT_RT)
  2894. +and are used in the quality management of the Linux real-time
  2895. +capabilities.
  2896. +
  2897. +
  2898. +* Purpose of latency histograms
  2899. +
  2900. +A latency histogram continuously accumulates the frequencies of latency
  2901. +data. There are two types of histograms
  2902. +- potential sources of latencies
  2903. +- effective latencies
  2904. +
  2905. +
  2906. +* Potential sources of latencies
  2907. +
  2908. +Potential sources of latencies are code segments where interrupts,
  2909. +preemption or both are disabled (aka critical sections). To create
  2910. +histograms of potential sources of latency, the kernel stores the time
  2911. +stamp at the start of a critical section, determines the time elapsed
  2912. +when the end of the section is reached, and increments the frequency
  2913. +counter of that latency value - irrespective of whether any concurrently
  2914. +running process is affected by latency or not.
  2915. +- Configuration items (in the Kernel hacking/Tracers submenu)
  2916. + CONFIG_INTERRUPT_OFF_LATENCY
  2917. + CONFIG_PREEMPT_OFF_LATENCY
  2918. +
  2919. +
  2920. +* Effective latencies
  2921. +
  2922. +Effective latencies are actually occuring during wakeup of a process. To
  2923. +determine effective latencies, the kernel stores the time stamp when a
  2924. +process is scheduled to be woken up, and determines the duration of the
  2925. +wakeup time shortly before control is passed over to this process. Note
  2926. +that the apparent latency in user space may be somewhat longer, since the
  2927. +process may be interrupted after control is passed over to it but before
  2928. +the execution in user space takes place. Simply measuring the interval
  2929. +between enqueuing and wakeup may also not appropriate in cases when a
  2930. +process is scheduled as a result of a timer expiration. The timer may have
  2931. +missed its deadline, e.g. due to disabled interrupts, but this latency
  2932. +would not be registered. Therefore, the offsets of missed timers are
  2933. +recorded in a separate histogram. If both wakeup latency and missed timer
  2934. +offsets are configured and enabled, a third histogram may be enabled that
  2935. +records the overall latency as a sum of the timer latency, if any, and the
  2936. +wakeup latency. This histogram is called "timerandwakeup".
  2937. +- Configuration items (in the Kernel hacking/Tracers submenu)
  2938. + CONFIG_WAKEUP_LATENCY
  2939. + CONFIG_MISSED_TIMER_OFSETS
  2940. +
  2941. +
  2942. +* Usage
  2943. +
  2944. +The interface to the administration of the latency histograms is located
  2945. +in the debugfs file system. To mount it, either enter
  2946. +
  2947. +mount -t sysfs nodev /sys
  2948. +mount -t debugfs nodev /sys/kernel/debug
  2949. +
  2950. +from shell command line level, or add
  2951. +
  2952. +nodev /sys sysfs defaults 0 0
  2953. +nodev /sys/kernel/debug debugfs defaults 0 0
  2954. +
  2955. +to the file /etc/fstab. All latency histogram related files are then
  2956. +available in the directory /sys/kernel/debug/tracing/latency_hist. A
  2957. +particular histogram type is enabled by writing non-zero to the related
  2958. +variable in the /sys/kernel/debug/tracing/latency_hist/enable directory.
  2959. +Select "preemptirqsoff" for the histograms of potential sources of
  2960. +latencies and "wakeup" for histograms of effective latencies etc. The
  2961. +histogram data - one per CPU - are available in the files
  2962. +
  2963. +/sys/kernel/debug/tracing/latency_hist/preemptoff/CPUx
  2964. +/sys/kernel/debug/tracing/latency_hist/irqsoff/CPUx
  2965. +/sys/kernel/debug/tracing/latency_hist/preemptirqsoff/CPUx
  2966. +/sys/kernel/debug/tracing/latency_hist/wakeup/CPUx
  2967. +/sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio/CPUx
  2968. +/sys/kernel/debug/tracing/latency_hist/missed_timer_offsets/CPUx
  2969. +/sys/kernel/debug/tracing/latency_hist/timerandwakeup/CPUx
  2970. +
  2971. +The histograms are reset by writing non-zero to the file "reset" in a
  2972. +particular latency directory. To reset all latency data, use
  2973. +
  2974. +#!/bin/sh
  2975. +
  2976. +TRACINGDIR=/sys/kernel/debug/tracing
  2977. +HISTDIR=$TRACINGDIR/latency_hist
  2978. +
  2979. +if test -d $HISTDIR
  2980. +then
  2981. + cd $HISTDIR
  2982. + for i in `find . | grep /reset$`
  2983. + do
  2984. + echo 1 >$i
  2985. + done
  2986. +fi
  2987. +
  2988. +
  2989. +* Data format
  2990. +
  2991. +Latency data are stored with a resolution of one microsecond. The
  2992. +maximum latency is 10,240 microseconds. The data are only valid, if the
  2993. +overflow register is empty. Every output line contains the latency in
  2994. +microseconds in the first row and the number of samples in the second
  2995. +row. To display only lines with a positive latency count, use, for
  2996. +example,
  2997. +
  2998. +grep -v " 0$" /sys/kernel/debug/tracing/latency_hist/preemptoff/CPU0
  2999. +
  3000. +#Minimum latency: 0 microseconds.
  3001. +#Average latency: 0 microseconds.
  3002. +#Maximum latency: 25 microseconds.
  3003. +#Total samples: 3104770694
  3004. +#There are 0 samples greater or equal than 10240 microseconds
  3005. +#usecs samples
  3006. + 0 2984486876
  3007. + 1 49843506
  3008. + 2 58219047
  3009. + 3 5348126
  3010. + 4 2187960
  3011. + 5 3388262
  3012. + 6 959289
  3013. + 7 208294
  3014. + 8 40420
  3015. + 9 4485
  3016. + 10 14918
  3017. + 11 18340
  3018. + 12 25052
  3019. + 13 19455
  3020. + 14 5602
  3021. + 15 969
  3022. + 16 47
  3023. + 17 18
  3024. + 18 14
  3025. + 19 1
  3026. + 20 3
  3027. + 21 2
  3028. + 22 5
  3029. + 23 2
  3030. + 25 1
  3031. +
  3032. +
  3033. +* Wakeup latency of a selected process
  3034. +
  3035. +To only collect wakeup latency data of a particular process, write the
  3036. +PID of the requested process to
  3037. +
  3038. +/sys/kernel/debug/tracing/latency_hist/wakeup/pid
  3039. +
  3040. +PIDs are not considered, if this variable is set to 0.
  3041. +
  3042. +
  3043. +* Details of the process with the highest wakeup latency so far
  3044. +
  3045. +Selected data of the process that suffered from the highest wakeup
  3046. +latency that occurred in a particular CPU are available in the file
  3047. +
  3048. +/sys/kernel/debug/tracing/latency_hist/wakeup/max_latency-CPUx.
  3049. +
  3050. +In addition, other relevant system data at the time when the
  3051. +latency occurred are given.
  3052. +
  3053. +The format of the data is (all in one line):
  3054. +<PID> <Priority> <Latency> (<Timeroffset>) <Command> \
  3055. +<- <PID> <Priority> <Command> <Timestamp>
  3056. +
  3057. +The value of <Timeroffset> is only relevant in the combined timer
  3058. +and wakeup latency recording. In the wakeup recording, it is
  3059. +always 0, in the missed_timer_offsets recording, it is the same
  3060. +as <Latency>.
  3061. +
  3062. +When retrospectively searching for the origin of a latency and
  3063. +tracing was not enabled, it may be helpful to know the name and
  3064. +some basic data of the task that (finally) was switching to the
  3065. +late real-tlme task. In addition to the victim's data, also the
  3066. +data of the possible culprit are therefore displayed after the
  3067. +"<-" symbol.
  3068. +
  3069. +Finally, the timestamp of the time when the latency occurred
  3070. +in <seconds>.<microseconds> after the most recent system boot
  3071. +is provided.
  3072. +
  3073. +These data are also reset when the wakeup histogram is reset.
  3074. diff -Nur linux-4.6.6.orig/drivers/acpi/acpica/acglobal.h linux-4.6.6/drivers/acpi/acpica/acglobal.h
  3075. --- linux-4.6.6.orig/drivers/acpi/acpica/acglobal.h 2016-08-10 12:54:57.000000000 +0200
  3076. +++ linux-4.6.6/drivers/acpi/acpica/acglobal.h 2016-08-22 21:27:46.733849506 +0200
  3077. @@ -116,7 +116,7 @@
  3078. * interrupt level
  3079. */
  3080. ACPI_GLOBAL(acpi_spinlock, acpi_gbl_gpe_lock); /* For GPE data structs and registers */
  3081. -ACPI_GLOBAL(acpi_spinlock, acpi_gbl_hardware_lock); /* For ACPI H/W except GPE registers */
  3082. +ACPI_GLOBAL(acpi_raw_spinlock, acpi_gbl_hardware_lock); /* For ACPI H/W except GPE registers */
  3083. ACPI_GLOBAL(acpi_spinlock, acpi_gbl_reference_count_lock);
  3084. /* Mutex for _OSI support */
  3085. diff -Nur linux-4.6.6.orig/drivers/acpi/acpica/hwregs.c linux-4.6.6/drivers/acpi/acpica/hwregs.c
  3086. --- linux-4.6.6.orig/drivers/acpi/acpica/hwregs.c 2016-08-10 12:54:57.000000000 +0200
  3087. +++ linux-4.6.6/drivers/acpi/acpica/hwregs.c 2016-08-22 21:27:46.741850019 +0200
  3088. @@ -269,14 +269,14 @@
  3089. ACPI_BITMASK_ALL_FIXED_STATUS,
  3090. ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address)));
  3091. - lock_flags = acpi_os_acquire_lock(acpi_gbl_hardware_lock);
  3092. + raw_spin_lock_irqsave(acpi_gbl_hardware_lock, lock_flags);
  3093. /* Clear the fixed events in PM1 A/B */
  3094. status = acpi_hw_register_write(ACPI_REGISTER_PM1_STATUS,
  3095. ACPI_BITMASK_ALL_FIXED_STATUS);
  3096. - acpi_os_release_lock(acpi_gbl_hardware_lock, lock_flags);
  3097. + raw_spin_unlock_irqrestore(acpi_gbl_hardware_lock, lock_flags);
  3098. if (ACPI_FAILURE(status)) {
  3099. goto exit;
  3100. diff -Nur linux-4.6.6.orig/drivers/acpi/acpica/hwxface.c linux-4.6.6/drivers/acpi/acpica/hwxface.c
  3101. --- linux-4.6.6.orig/drivers/acpi/acpica/hwxface.c 2016-08-10 12:54:57.000000000 +0200
  3102. +++ linux-4.6.6/drivers/acpi/acpica/hwxface.c 2016-08-22 21:27:46.745850272 +0200
  3103. @@ -374,7 +374,7 @@
  3104. return_ACPI_STATUS(AE_BAD_PARAMETER);
  3105. }
  3106. - lock_flags = acpi_os_acquire_lock(acpi_gbl_hardware_lock);
  3107. + raw_spin_lock_irqsave(acpi_gbl_hardware_lock, lock_flags);
  3108. /*
  3109. * At this point, we know that the parent register is one of the
  3110. @@ -435,7 +435,7 @@
  3111. unlock_and_exit:
  3112. - acpi_os_release_lock(acpi_gbl_hardware_lock, lock_flags);
  3113. + raw_spin_unlock_irqrestore(acpi_gbl_hardware_lock, lock_flags);
  3114. return_ACPI_STATUS(status);
  3115. }
  3116. diff -Nur linux-4.6.6.orig/drivers/acpi/acpica/utmutex.c linux-4.6.6/drivers/acpi/acpica/utmutex.c
  3117. --- linux-4.6.6.orig/drivers/acpi/acpica/utmutex.c 2016-08-10 12:54:57.000000000 +0200
  3118. +++ linux-4.6.6/drivers/acpi/acpica/utmutex.c 2016-08-22 21:27:46.769851812 +0200
  3119. @@ -88,7 +88,7 @@
  3120. return_ACPI_STATUS (status);
  3121. }
  3122. - status = acpi_os_create_lock (&acpi_gbl_hardware_lock);
  3123. + status = acpi_os_create_raw_lock (&acpi_gbl_hardware_lock);
  3124. if (ACPI_FAILURE (status)) {
  3125. return_ACPI_STATUS (status);
  3126. }
  3127. @@ -145,7 +145,7 @@
  3128. /* Delete the spinlocks */
  3129. acpi_os_delete_lock(acpi_gbl_gpe_lock);
  3130. - acpi_os_delete_lock(acpi_gbl_hardware_lock);
  3131. + acpi_os_delete_raw_lock(acpi_gbl_hardware_lock);
  3132. acpi_os_delete_lock(acpi_gbl_reference_count_lock);
  3133. /* Delete the reader/writer lock */
  3134. diff -Nur linux-4.6.6.orig/drivers/ata/libata-sff.c linux-4.6.6/drivers/ata/libata-sff.c
  3135. --- linux-4.6.6.orig/drivers/ata/libata-sff.c 2016-08-10 12:54:57.000000000 +0200
  3136. +++ linux-4.6.6/drivers/ata/libata-sff.c 2016-08-22 21:27:46.781852585 +0200
  3137. @@ -678,9 +678,9 @@
  3138. unsigned long flags;
  3139. unsigned int consumed;
  3140. - local_irq_save(flags);
  3141. + local_irq_save_nort(flags);
  3142. consumed = ata_sff_data_xfer32(dev, buf, buflen, rw);
  3143. - local_irq_restore(flags);
  3144. + local_irq_restore_nort(flags);
  3145. return consumed;
  3146. }
  3147. @@ -719,7 +719,7 @@
  3148. unsigned long flags;
  3149. /* FIXME: use a bounce buffer */
  3150. - local_irq_save(flags);
  3151. + local_irq_save_nort(flags);
  3152. buf = kmap_atomic(page);
  3153. /* do the actual data transfer */
  3154. @@ -727,7 +727,7 @@
  3155. do_write);
  3156. kunmap_atomic(buf);
  3157. - local_irq_restore(flags);
  3158. + local_irq_restore_nort(flags);
  3159. } else {
  3160. buf = page_address(page);
  3161. ap->ops->sff_data_xfer(qc->dev, buf + offset, qc->sect_size,
  3162. @@ -864,7 +864,7 @@
  3163. unsigned long flags;
  3164. /* FIXME: use bounce buffer */
  3165. - local_irq_save(flags);
  3166. + local_irq_save_nort(flags);
  3167. buf = kmap_atomic(page);
  3168. /* do the actual data transfer */
  3169. @@ -872,7 +872,7 @@
  3170. count, rw);
  3171. kunmap_atomic(buf);
  3172. - local_irq_restore(flags);
  3173. + local_irq_restore_nort(flags);
  3174. } else {
  3175. buf = page_address(page);
  3176. consumed = ap->ops->sff_data_xfer(dev, buf + offset,
  3177. diff -Nur linux-4.6.6.orig/drivers/block/zram/zram_drv.c linux-4.6.6/drivers/block/zram/zram_drv.c
  3178. --- linux-4.6.6.orig/drivers/block/zram/zram_drv.c 2016-08-10 12:54:57.000000000 +0200
  3179. +++ linux-4.6.6/drivers/block/zram/zram_drv.c 2016-08-22 21:27:46.789853098 +0200
  3180. @@ -520,6 +520,8 @@
  3181. goto out_error;
  3182. }
  3183. + zram_meta_init_table_locks(meta, disksize);
  3184. +
  3185. return meta;
  3186. out_error:
  3187. @@ -568,12 +570,12 @@
  3188. unsigned long handle;
  3189. size_t size;
  3190. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3191. + zram_lock_table(&meta->table[index]);
  3192. handle = meta->table[index].handle;
  3193. size = zram_get_obj_size(meta, index);
  3194. if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
  3195. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3196. + zram_unlock_table(&meta->table[index]);
  3197. clear_page(mem);
  3198. return 0;
  3199. }
  3200. @@ -584,7 +586,7 @@
  3201. else
  3202. ret = zcomp_decompress(zram->comp, cmem, size, mem);
  3203. zs_unmap_object(meta->mem_pool, handle);
  3204. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3205. + zram_unlock_table(&meta->table[index]);
  3206. /* Should NEVER happen. Return bio error if it does. */
  3207. if (unlikely(ret)) {
  3208. @@ -604,14 +606,14 @@
  3209. struct zram_meta *meta = zram->meta;
  3210. page = bvec->bv_page;
  3211. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3212. + zram_lock_table(&meta->table[index]);
  3213. if (unlikely(!meta->table[index].handle) ||
  3214. zram_test_flag(meta, index, ZRAM_ZERO)) {
  3215. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3216. + zram_unlock_table(&meta->table[index]);
  3217. handle_zero_page(bvec);
  3218. return 0;
  3219. }
  3220. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3221. + zram_unlock_table(&meta->table[index]);
  3222. if (is_partial_io(bvec))
  3223. /* Use a temporary buffer to decompress the page */
  3224. @@ -689,10 +691,10 @@
  3225. if (user_mem)
  3226. kunmap_atomic(user_mem);
  3227. /* Free memory associated with this sector now. */
  3228. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3229. + zram_lock_table(&meta->table[index]);
  3230. zram_free_page(zram, index);
  3231. zram_set_flag(meta, index, ZRAM_ZERO);
  3232. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3233. + zram_unlock_table(&meta->table[index]);
  3234. atomic64_inc(&zram->stats.zero_pages);
  3235. ret = 0;
  3236. @@ -752,12 +754,12 @@
  3237. * Free memory associated with this sector
  3238. * before overwriting unused sectors.
  3239. */
  3240. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3241. + zram_lock_table(&meta->table[index]);
  3242. zram_free_page(zram, index);
  3243. meta->table[index].handle = handle;
  3244. zram_set_obj_size(meta, index, clen);
  3245. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3246. + zram_unlock_table(&meta->table[index]);
  3247. /* Update stats */
  3248. atomic64_add(clen, &zram->stats.compr_data_size);
  3249. @@ -800,9 +802,9 @@
  3250. }
  3251. while (n >= PAGE_SIZE) {
  3252. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3253. + zram_lock_table(&meta->table[index]);
  3254. zram_free_page(zram, index);
  3255. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3256. + zram_unlock_table(&meta->table[index]);
  3257. atomic64_inc(&zram->stats.notify_free);
  3258. index++;
  3259. n -= PAGE_SIZE;
  3260. @@ -928,9 +930,9 @@
  3261. zram = bdev->bd_disk->private_data;
  3262. meta = zram->meta;
  3263. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3264. + zram_lock_table(&meta->table[index]);
  3265. zram_free_page(zram, index);
  3266. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3267. + zram_unlock_table(&meta->table[index]);
  3268. atomic64_inc(&zram->stats.notify_free);
  3269. }
  3270. diff -Nur linux-4.6.6.orig/drivers/block/zram/zram_drv.h linux-4.6.6/drivers/block/zram/zram_drv.h
  3271. --- linux-4.6.6.orig/drivers/block/zram/zram_drv.h 2016-08-10 12:54:57.000000000 +0200
  3272. +++ linux-4.6.6/drivers/block/zram/zram_drv.h 2016-08-22 21:27:46.789853098 +0200
  3273. @@ -72,6 +72,9 @@
  3274. struct zram_table_entry {
  3275. unsigned long handle;
  3276. unsigned long value;
  3277. +#ifdef CONFIG_PREEMPT_RT_BASE
  3278. + spinlock_t lock;
  3279. +#endif
  3280. };
  3281. struct zram_stats {
  3282. @@ -119,4 +122,42 @@
  3283. */
  3284. bool claim; /* Protected by bdev->bd_mutex */
  3285. };
  3286. +
  3287. +#ifndef CONFIG_PREEMPT_RT_BASE
  3288. +static inline void zram_lock_table(struct zram_table_entry *table)
  3289. +{
  3290. + bit_spin_lock(ZRAM_ACCESS, &table->value);
  3291. +}
  3292. +
  3293. +static inline void zram_unlock_table(struct zram_table_entry *table)
  3294. +{
  3295. + bit_spin_unlock(ZRAM_ACCESS, &table->value);
  3296. +}
  3297. +
  3298. +static inline void zram_meta_init_table_locks(struct zram_meta *meta, u64 disksize) { }
  3299. +#else /* CONFIG_PREEMPT_RT_BASE */
  3300. +static inline void zram_lock_table(struct zram_table_entry *table)
  3301. +{
  3302. + spin_lock(&table->lock);
  3303. + __set_bit(ZRAM_ACCESS, &table->value);
  3304. +}
  3305. +
  3306. +static inline void zram_unlock_table(struct zram_table_entry *table)
  3307. +{
  3308. + __clear_bit(ZRAM_ACCESS, &table->value);
  3309. + spin_unlock(&table->lock);
  3310. +}
  3311. +
  3312. +static inline void zram_meta_init_table_locks(struct zram_meta *meta, u64 disksize)
  3313. +{
  3314. + size_t num_pages = disksize >> PAGE_SHIFT;
  3315. + size_t index;
  3316. +
  3317. + for (index = 0; index < num_pages; index++) {
  3318. + spinlock_t *lock = &meta->table[index].lock;
  3319. + spin_lock_init(lock);
  3320. + }
  3321. +}
  3322. +#endif /* CONFIG_PREEMPT_RT_BASE */
  3323. +
  3324. #endif
  3325. diff -Nur linux-4.6.6.orig/drivers/char/random.c linux-4.6.6/drivers/char/random.c
  3326. --- linux-4.6.6.orig/drivers/char/random.c 2016-08-10 12:54:57.000000000 +0200
  3327. +++ linux-4.6.6/drivers/char/random.c 2016-08-22 21:27:46.797853611 +0200
  3328. @@ -796,8 +796,6 @@
  3329. } sample;
  3330. long delta, delta2, delta3;
  3331. - preempt_disable();
  3332. -
  3333. sample.jiffies = jiffies;
  3334. sample.cycles = random_get_entropy();
  3335. sample.num = num;
  3336. @@ -838,7 +836,6 @@
  3337. */
  3338. credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
  3339. }
  3340. - preempt_enable();
  3341. }
  3342. void add_input_randomness(unsigned int type, unsigned int code,
  3343. @@ -891,28 +888,27 @@
  3344. return *(ptr + f->reg_idx++);
  3345. }
  3346. -void add_interrupt_randomness(int irq, int irq_flags)
  3347. +void add_interrupt_randomness(int irq, int irq_flags, __u64 ip)
  3348. {
  3349. struct entropy_store *r;
  3350. struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
  3351. - struct pt_regs *regs = get_irq_regs();
  3352. unsigned long now = jiffies;
  3353. cycles_t cycles = random_get_entropy();
  3354. __u32 c_high, j_high;
  3355. - __u64 ip;
  3356. unsigned long seed;
  3357. int credit = 0;
  3358. if (cycles == 0)
  3359. - cycles = get_reg(fast_pool, regs);
  3360. + cycles = get_reg(fast_pool, NULL);
  3361. c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
  3362. j_high = (sizeof(now) > 4) ? now >> 32 : 0;
  3363. fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
  3364. fast_pool->pool[1] ^= now ^ c_high;
  3365. - ip = regs ? instruction_pointer(regs) : _RET_IP_;
  3366. + if (!ip)
  3367. + ip = _RET_IP_;
  3368. fast_pool->pool[2] ^= ip;
  3369. fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
  3370. - get_reg(fast_pool, regs);
  3371. + get_reg(fast_pool, NULL);
  3372. fast_mix(fast_pool);
  3373. add_interrupt_bench(cycles);
  3374. diff -Nur linux-4.6.6.orig/drivers/clocksource/tcb_clksrc.c linux-4.6.6/drivers/clocksource/tcb_clksrc.c
  3375. --- linux-4.6.6.orig/drivers/clocksource/tcb_clksrc.c 2016-08-10 12:54:57.000000000 +0200
  3376. +++ linux-4.6.6/drivers/clocksource/tcb_clksrc.c 2016-08-22 21:27:46.805854124 +0200
  3377. @@ -23,8 +23,7 @@
  3378. * this 32 bit free-running counter. the second channel is not used.
  3379. *
  3380. * - The third channel may be used to provide a 16-bit clockevent
  3381. - * source, used in either periodic or oneshot mode. This runs
  3382. - * at 32 KiHZ, and can handle delays of up to two seconds.
  3383. + * source, used in either periodic or oneshot mode.
  3384. *
  3385. * A boot clocksource and clockevent source are also currently needed,
  3386. * unless the relevant platforms (ARM/AT91, AVR32/AT32) are changed so
  3387. @@ -74,6 +73,8 @@
  3388. struct tc_clkevt_device {
  3389. struct clock_event_device clkevt;
  3390. struct clk *clk;
  3391. + bool clk_enabled;
  3392. + u32 freq;
  3393. void __iomem *regs;
  3394. };
  3395. @@ -82,15 +83,26 @@
  3396. return container_of(clkevt, struct tc_clkevt_device, clkevt);
  3397. }
  3398. -/* For now, we always use the 32K clock ... this optimizes for NO_HZ,
  3399. - * because using one of the divided clocks would usually mean the
  3400. - * tick rate can never be less than several dozen Hz (vs 0.5 Hz).
  3401. - *
  3402. - * A divided clock could be good for high resolution timers, since
  3403. - * 30.5 usec resolution can seem "low".
  3404. - */
  3405. static u32 timer_clock;
  3406. +static void tc_clk_disable(struct clock_event_device *d)
  3407. +{
  3408. + struct tc_clkevt_device *tcd = to_tc_clkevt(d);
  3409. +
  3410. + clk_disable(tcd->clk);
  3411. + tcd->clk_enabled = false;
  3412. +}
  3413. +
  3414. +static void tc_clk_enable(struct clock_event_device *d)
  3415. +{
  3416. + struct tc_clkevt_device *tcd = to_tc_clkevt(d);
  3417. +
  3418. + if (tcd->clk_enabled)
  3419. + return;
  3420. + clk_enable(tcd->clk);
  3421. + tcd->clk_enabled = true;
  3422. +}
  3423. +
  3424. static int tc_shutdown(struct clock_event_device *d)
  3425. {
  3426. struct tc_clkevt_device *tcd = to_tc_clkevt(d);
  3427. @@ -98,8 +110,14 @@
  3428. __raw_writel(0xff, regs + ATMEL_TC_REG(2, IDR));
  3429. __raw_writel(ATMEL_TC_CLKDIS, regs + ATMEL_TC_REG(2, CCR));
  3430. + return 0;
  3431. +}
  3432. +
  3433. +static int tc_shutdown_clk_off(struct clock_event_device *d)
  3434. +{
  3435. + tc_shutdown(d);
  3436. if (!clockevent_state_detached(d))
  3437. - clk_disable(tcd->clk);
  3438. + tc_clk_disable(d);
  3439. return 0;
  3440. }
  3441. @@ -112,9 +130,9 @@
  3442. if (clockevent_state_oneshot(d) || clockevent_state_periodic(d))
  3443. tc_shutdown(d);
  3444. - clk_enable(tcd->clk);
  3445. + tc_clk_enable(d);
  3446. - /* slow clock, count up to RC, then irq and stop */
  3447. + /* count up to RC, then irq and stop */
  3448. __raw_writel(timer_clock | ATMEL_TC_CPCSTOP | ATMEL_TC_WAVE |
  3449. ATMEL_TC_WAVESEL_UP_AUTO, regs + ATMEL_TC_REG(2, CMR));
  3450. __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER));
  3451. @@ -134,12 +152,12 @@
  3452. /* By not making the gentime core emulate periodic mode on top
  3453. * of oneshot, we get lower overhead and improved accuracy.
  3454. */
  3455. - clk_enable(tcd->clk);
  3456. + tc_clk_enable(d);
  3457. - /* slow clock, count up to RC, then irq and restart */
  3458. + /* count up to RC, then irq and restart */
  3459. __raw_writel(timer_clock | ATMEL_TC_WAVE | ATMEL_TC_WAVESEL_UP_AUTO,
  3460. regs + ATMEL_TC_REG(2, CMR));
  3461. - __raw_writel((32768 + HZ / 2) / HZ, tcaddr + ATMEL_TC_REG(2, RC));
  3462. + __raw_writel((tcd->freq + HZ / 2) / HZ, tcaddr + ATMEL_TC_REG(2, RC));
  3463. /* Enable clock and interrupts on RC compare */
  3464. __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER));
  3465. @@ -166,9 +184,13 @@
  3466. .features = CLOCK_EVT_FEAT_PERIODIC |
  3467. CLOCK_EVT_FEAT_ONESHOT,
  3468. /* Should be lower than at91rm9200's system timer */
  3469. +#ifdef CONFIG_ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK
  3470. .rating = 125,
  3471. +#else
  3472. + .rating = 200,
  3473. +#endif
  3474. .set_next_event = tc_next_event,
  3475. - .set_state_shutdown = tc_shutdown,
  3476. + .set_state_shutdown = tc_shutdown_clk_off,
  3477. .set_state_periodic = tc_set_periodic,
  3478. .set_state_oneshot = tc_set_oneshot,
  3479. },
  3480. @@ -188,8 +210,9 @@
  3481. return IRQ_NONE;
  3482. }
  3483. -static int __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx)
  3484. +static int __init setup_clkevents(struct atmel_tc *tc, int divisor_idx)
  3485. {
  3486. + unsigned divisor = atmel_tc_divisors[divisor_idx];
  3487. int ret;
  3488. struct clk *t2_clk = tc->clk[2];
  3489. int irq = tc->irq[2];
  3490. @@ -210,7 +233,11 @@
  3491. clkevt.regs = tc->regs;
  3492. clkevt.clk = t2_clk;
  3493. - timer_clock = clk32k_divisor_idx;
  3494. + timer_clock = divisor_idx;
  3495. + if (!divisor)
  3496. + clkevt.freq = 32768;
  3497. + else
  3498. + clkevt.freq = clk_get_rate(t2_clk) / divisor;
  3499. clkevt.clkevt.cpumask = cpumask_of(0);
  3500. @@ -221,7 +248,7 @@
  3501. return ret;
  3502. }
  3503. - clockevents_config_and_register(&clkevt.clkevt, 32768, 1, 0xffff);
  3504. + clockevents_config_and_register(&clkevt.clkevt, clkevt.freq, 1, 0xffff);
  3505. return ret;
  3506. }
  3507. @@ -358,7 +385,11 @@
  3508. goto err_disable_t1;
  3509. /* channel 2: periodic and oneshot timer support */
  3510. +#ifdef CONFIG_ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK
  3511. ret = setup_clkevents(tc, clk32k_divisor_idx);
  3512. +#else
  3513. + ret = setup_clkevents(tc, best_divisor_idx);
  3514. +#endif
  3515. if (ret)
  3516. goto err_unregister_clksrc;
  3517. diff -Nur linux-4.6.6.orig/drivers/clocksource/timer-atmel-pit.c linux-4.6.6/drivers/clocksource/timer-atmel-pit.c
  3518. --- linux-4.6.6.orig/drivers/clocksource/timer-atmel-pit.c 2016-08-10 12:54:57.000000000 +0200
  3519. +++ linux-4.6.6/drivers/clocksource/timer-atmel-pit.c 2016-08-22 21:27:46.805854124 +0200
  3520. @@ -46,6 +46,7 @@
  3521. u32 cycle;
  3522. u32 cnt;
  3523. unsigned int irq;
  3524. + bool irq_requested;
  3525. struct clk *mck;
  3526. };
  3527. @@ -96,15 +97,29 @@
  3528. /* disable irq, leaving the clocksource active */
  3529. pit_write(data->base, AT91_PIT_MR, (data->cycle - 1) | AT91_PIT_PITEN);
  3530. + if (data->irq_requested) {
  3531. + free_irq(data->irq, data);
  3532. + data->irq_requested = false;
  3533. + }
  3534. return 0;
  3535. }
  3536. +static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id);
  3537. /*
  3538. * Clockevent device: interrupts every 1/HZ (== pit_cycles * MCK/16)
  3539. */
  3540. static int pit_clkevt_set_periodic(struct clock_event_device *dev)
  3541. {
  3542. struct pit_data *data = clkevt_to_pit_data(dev);
  3543. + int ret;
  3544. +
  3545. + ret = request_irq(data->irq, at91sam926x_pit_interrupt,
  3546. + IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3547. + "at91_tick", data);
  3548. + if (ret)
  3549. + panic(pr_fmt("Unable to setup IRQ\n"));
  3550. +
  3551. + data->irq_requested = true;
  3552. /* update clocksource counter */
  3553. data->cnt += data->cycle * PIT_PICNT(pit_read(data->base, AT91_PIT_PIVR));
  3554. @@ -181,7 +196,6 @@
  3555. {
  3556. unsigned long pit_rate;
  3557. unsigned bits;
  3558. - int ret;
  3559. /*
  3560. * Use our actual MCK to figure out how many MCK/16 ticks per
  3561. @@ -206,13 +220,6 @@
  3562. data->clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS;
  3563. clocksource_register_hz(&data->clksrc, pit_rate);
  3564. - /* Set up irq handler */
  3565. - ret = request_irq(data->irq, at91sam926x_pit_interrupt,
  3566. - IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3567. - "at91_tick", data);
  3568. - if (ret)
  3569. - panic(pr_fmt("Unable to setup IRQ\n"));
  3570. -
  3571. /* Set up and register clockevents */
  3572. data->clkevt.name = "pit";
  3573. data->clkevt.features = CLOCK_EVT_FEAT_PERIODIC;
  3574. diff -Nur linux-4.6.6.orig/drivers/clocksource/timer-atmel-st.c linux-4.6.6/drivers/clocksource/timer-atmel-st.c
  3575. --- linux-4.6.6.orig/drivers/clocksource/timer-atmel-st.c 2016-08-10 12:54:57.000000000 +0200
  3576. +++ linux-4.6.6/drivers/clocksource/timer-atmel-st.c 2016-08-22 21:27:46.805854124 +0200
  3577. @@ -115,18 +115,29 @@
  3578. last_crtr = read_CRTR();
  3579. }
  3580. +static int atmel_st_irq;
  3581. +
  3582. static int clkevt32k_shutdown(struct clock_event_device *evt)
  3583. {
  3584. clkdev32k_disable_and_flush_irq();
  3585. irqmask = 0;
  3586. regmap_write(regmap_st, AT91_ST_IER, irqmask);
  3587. + free_irq(atmel_st_irq, regmap_st);
  3588. return 0;
  3589. }
  3590. static int clkevt32k_set_oneshot(struct clock_event_device *dev)
  3591. {
  3592. + int ret;
  3593. +
  3594. clkdev32k_disable_and_flush_irq();
  3595. + ret = request_irq(atmel_st_irq, at91rm9200_timer_interrupt,
  3596. + IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3597. + "at91_tick", regmap_st);
  3598. + if (ret)
  3599. + panic(pr_fmt("Unable to setup IRQ\n"));
  3600. +
  3601. /*
  3602. * ALM for oneshot irqs, set by next_event()
  3603. * before 32 seconds have passed.
  3604. @@ -139,8 +150,16 @@
  3605. static int clkevt32k_set_periodic(struct clock_event_device *dev)
  3606. {
  3607. + int ret;
  3608. +
  3609. clkdev32k_disable_and_flush_irq();
  3610. + ret = request_irq(atmel_st_irq, at91rm9200_timer_interrupt,
  3611. + IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3612. + "at91_tick", regmap_st);
  3613. + if (ret)
  3614. + panic(pr_fmt("Unable to setup IRQ\n"));
  3615. +
  3616. /* PIT for periodic irqs; fixed rate of 1/HZ */
  3617. irqmask = AT91_ST_PITS;
  3618. regmap_write(regmap_st, AT91_ST_PIMR, timer_latch);
  3619. @@ -198,7 +217,7 @@
  3620. {
  3621. struct clk *sclk;
  3622. unsigned int sclk_rate, val;
  3623. - int irq, ret;
  3624. + int ret;
  3625. regmap_st = syscon_node_to_regmap(node);
  3626. if (IS_ERR(regmap_st))
  3627. @@ -210,17 +229,10 @@
  3628. regmap_read(regmap_st, AT91_ST_SR, &val);
  3629. /* Get the interrupts property */
  3630. - irq = irq_of_parse_and_map(node, 0);
  3631. - if (!irq)
  3632. + atmel_st_irq = irq_of_parse_and_map(node, 0);
  3633. + if (!atmel_st_irq)
  3634. panic(pr_fmt("Unable to get IRQ from DT\n"));
  3635. - /* Make IRQs happen for the system timer */
  3636. - ret = request_irq(irq, at91rm9200_timer_interrupt,
  3637. - IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3638. - "at91_tick", regmap_st);
  3639. - if (ret)
  3640. - panic(pr_fmt("Unable to setup IRQ\n"));
  3641. -
  3642. sclk = of_clk_get(node, 0);
  3643. if (IS_ERR(sclk))
  3644. panic(pr_fmt("Unable to get slow clock\n"));
  3645. diff -Nur linux-4.6.6.orig/drivers/cpufreq/Kconfig.x86 linux-4.6.6/drivers/cpufreq/Kconfig.x86
  3646. --- linux-4.6.6.orig/drivers/cpufreq/Kconfig.x86 2016-08-10 12:54:57.000000000 +0200
  3647. +++ linux-4.6.6/drivers/cpufreq/Kconfig.x86 2016-08-22 21:27:46.809854377 +0200
  3648. @@ -123,7 +123,7 @@
  3649. config X86_POWERNOW_K8
  3650. tristate "AMD Opteron/Athlon64 PowerNow!"
  3651. - depends on ACPI && ACPI_PROCESSOR && X86_ACPI_CPUFREQ
  3652. + depends on ACPI && ACPI_PROCESSOR && X86_ACPI_CPUFREQ && !PREEMPT_RT_BASE
  3653. help
  3654. This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors.
  3655. Support for K10 and newer processors is now in acpi-cpufreq.
  3656. diff -Nur linux-4.6.6.orig/drivers/crypto/ccp/ccp-dev.c linux-4.6.6/drivers/crypto/ccp/ccp-dev.c
  3657. --- linux-4.6.6.orig/drivers/crypto/ccp/ccp-dev.c 2016-08-10 12:54:57.000000000 +0200
  3658. +++ linux-4.6.6/drivers/crypto/ccp/ccp-dev.c 2016-08-22 21:27:46.813854637 +0200
  3659. @@ -16,7 +16,6 @@
  3660. #include <linux/sched.h>
  3661. #include <linux/interrupt.h>
  3662. #include <linux/spinlock.h>
  3663. -#include <linux/rwlock_types.h>
  3664. #include <linux/types.h>
  3665. #include <linux/mutex.h>
  3666. #include <linux/delay.h>
  3667. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/i915_drv.h linux-4.6.6/drivers/gpu/drm/i915/i915_drv.h
  3668. --- linux-4.6.6.orig/drivers/gpu/drm/i915/i915_drv.h 2016-08-10 12:54:57.000000000 +0200
  3669. +++ linux-4.6.6/drivers/gpu/drm/i915/i915_drv.h 2016-08-22 21:27:46.833855915 +0200
  3670. @@ -714,7 +714,7 @@
  3671. struct drm_i915_private *i915;
  3672. enum forcewake_domain_id id;
  3673. unsigned wake_count;
  3674. - struct timer_list timer;
  3675. + struct hrtimer timer;
  3676. i915_reg_t reg_set;
  3677. u32 val_set;
  3678. u32 val_clear;
  3679. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.6.6/drivers/gpu/drm/i915/i915_gem_execbuffer.c
  3680. --- linux-4.6.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2016-08-10 12:54:57.000000000 +0200
  3681. +++ linux-4.6.6/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2016-08-22 21:27:46.845856688 +0200
  3682. @@ -1314,7 +1314,9 @@
  3683. if (ret)
  3684. return ret;
  3685. +#ifndef CONFIG_PREEMPT_RT_BASE
  3686. trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
  3687. +#endif
  3688. i915_gem_execbuffer_move_to_active(vmas, params->request);
  3689. i915_gem_execbuffer_retire_commands(params);
  3690. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.6.6/drivers/gpu/drm/i915/i915_gem_shrinker.c
  3691. --- linux-4.6.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2016-08-10 12:54:57.000000000 +0200
  3692. +++ linux-4.6.6/drivers/gpu/drm/i915/i915_gem_shrinker.c 2016-08-22 21:27:46.845856688 +0200
  3693. @@ -39,7 +39,7 @@
  3694. if (!mutex_is_locked(mutex))
  3695. return false;
  3696. -#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)
  3697. +#if (defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)) && !defined(CONFIG_PREEMPT_RT_BASE)
  3698. return mutex->owner == task;
  3699. #else
  3700. /* Since UP may be pre-empted, we cannot assume that we own the lock */
  3701. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/i915_irq.c linux-4.6.6/drivers/gpu/drm/i915/i915_irq.c
  3702. --- linux-4.6.6.orig/drivers/gpu/drm/i915/i915_irq.c 2016-08-10 12:54:57.000000000 +0200
  3703. +++ linux-4.6.6/drivers/gpu/drm/i915/i915_irq.c 2016-08-22 21:27:46.865857967 +0200
  3704. @@ -830,6 +830,7 @@
  3705. spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
  3706. /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
  3707. + preempt_disable_rt();
  3708. /* Get optional system timestamp before query. */
  3709. if (stime)
  3710. @@ -881,6 +882,7 @@
  3711. *etime = ktime_get();
  3712. /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
  3713. + preempt_enable_rt();
  3714. spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
  3715. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/intel_display.c linux-4.6.6/drivers/gpu/drm/i915/intel_display.c
  3716. --- linux-4.6.6.orig/drivers/gpu/drm/i915/intel_display.c 2016-08-10 12:54:57.000000000 +0200
  3717. +++ linux-4.6.6/drivers/gpu/drm/i915/intel_display.c 2016-08-22 21:27:46.905860533 +0200
  3718. @@ -11496,7 +11496,7 @@
  3719. struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
  3720. struct intel_unpin_work *work;
  3721. - WARN_ON(!in_interrupt());
  3722. + WARN_ON_NONRT(!in_interrupt());
  3723. if (crtc == NULL)
  3724. return;
  3725. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/intel_sprite.c linux-4.6.6/drivers/gpu/drm/i915/intel_sprite.c
  3726. --- linux-4.6.6.orig/drivers/gpu/drm/i915/intel_sprite.c 2016-08-10 12:54:57.000000000 +0200
  3727. +++ linux-4.6.6/drivers/gpu/drm/i915/intel_sprite.c 2016-08-22 21:27:46.917861306 +0200
  3728. @@ -38,6 +38,7 @@
  3729. #include "intel_drv.h"
  3730. #include <drm/i915_drm.h>
  3731. #include "i915_drv.h"
  3732. +#include <linux/locallock.h>
  3733. static bool
  3734. format_is_yuv(uint32_t format)
  3735. @@ -64,6 +65,8 @@
  3736. 1000 * adjusted_mode->crtc_htotal);
  3737. }
  3738. +static DEFINE_LOCAL_IRQ_LOCK(pipe_update_lock);
  3739. +
  3740. /**
  3741. * intel_pipe_update_start() - start update of a set of display registers
  3742. * @crtc: the crtc of which the registers are going to be updated
  3743. @@ -96,7 +99,7 @@
  3744. min = vblank_start - usecs_to_scanlines(adjusted_mode, 100);
  3745. max = vblank_start - 1;
  3746. - local_irq_disable();
  3747. + local_lock_irq(pipe_update_lock);
  3748. if (min <= 0 || max <= 0)
  3749. return;
  3750. @@ -126,11 +129,11 @@
  3751. break;
  3752. }
  3753. - local_irq_enable();
  3754. + local_unlock_irq(pipe_update_lock);
  3755. timeout = schedule_timeout(timeout);
  3756. - local_irq_disable();
  3757. + local_lock_irq(pipe_update_lock);
  3758. }
  3759. finish_wait(wq, &wait);
  3760. @@ -164,7 +167,7 @@
  3761. trace_i915_pipe_update_end(crtc, end_vbl_count, scanline_end);
  3762. - local_irq_enable();
  3763. + local_unlock_irq(pipe_update_lock);
  3764. if (crtc->debug.start_vbl_count &&
  3765. crtc->debug.start_vbl_count != end_vbl_count) {
  3766. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/i915/intel_uncore.c linux-4.6.6/drivers/gpu/drm/i915/intel_uncore.c
  3767. --- linux-4.6.6.orig/drivers/gpu/drm/i915/intel_uncore.c 2016-08-10 12:54:57.000000000 +0200
  3768. +++ linux-4.6.6/drivers/gpu/drm/i915/intel_uncore.c 2016-08-22 21:27:46.921861559 +0200
  3769. @@ -60,7 +60,11 @@
  3770. static inline void
  3771. fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
  3772. {
  3773. - mod_timer_pinned(&d->timer, jiffies + 1);
  3774. + d->wake_count++;
  3775. + hrtimer_start_range_ns(&d->timer,
  3776. + ktime_set(0, NSEC_PER_MSEC),
  3777. + NSEC_PER_MSEC,
  3778. + HRTIMER_MODE_REL);
  3779. }
  3780. static inline void
  3781. @@ -224,9 +228,11 @@
  3782. return ret;
  3783. }
  3784. -static void intel_uncore_fw_release_timer(unsigned long arg)
  3785. +static enum hrtimer_restart
  3786. +intel_uncore_fw_release_timer(struct hrtimer *timer)
  3787. {
  3788. - struct intel_uncore_forcewake_domain *domain = (void *)arg;
  3789. + struct intel_uncore_forcewake_domain *domain =
  3790. + container_of(timer, struct intel_uncore_forcewake_domain, timer);
  3791. unsigned long irqflags;
  3792. assert_rpm_device_not_suspended(domain->i915);
  3793. @@ -240,6 +246,8 @@
  3794. 1 << domain->id);
  3795. spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);
  3796. +
  3797. + return HRTIMER_NORESTART;
  3798. }
  3799. void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
  3800. @@ -259,16 +267,16 @@
  3801. active_domains = 0;
  3802. for_each_fw_domain(domain, dev_priv, id) {
  3803. - if (del_timer_sync(&domain->timer) == 0)
  3804. + if (hrtimer_cancel(&domain->timer) == 0)
  3805. continue;
  3806. - intel_uncore_fw_release_timer((unsigned long)domain);
  3807. + intel_uncore_fw_release_timer(&domain->timer);
  3808. }
  3809. spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
  3810. for_each_fw_domain(domain, dev_priv, id) {
  3811. - if (timer_pending(&domain->timer))
  3812. + if (hrtimer_active(&domain->timer))
  3813. active_domains |= (1 << id);
  3814. }
  3815. @@ -491,7 +499,6 @@
  3816. if (--domain->wake_count)
  3817. continue;
  3818. - domain->wake_count++;
  3819. fw_domain_arm_timer(domain);
  3820. }
  3821. }
  3822. @@ -732,7 +739,6 @@
  3823. continue;
  3824. }
  3825. - domain->wake_count++;
  3826. fw_domain_arm_timer(domain);
  3827. }
  3828. @@ -1150,7 +1156,8 @@
  3829. d->i915 = dev_priv;
  3830. d->id = domain_id;
  3831. - setup_timer(&d->timer, intel_uncore_fw_release_timer, (unsigned long)d);
  3832. + hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  3833. + d->timer.function = intel_uncore_fw_release_timer;
  3834. dev_priv->uncore.fw_domains |= (1 << domain_id);
  3835. diff -Nur linux-4.6.6.orig/drivers/gpu/drm/radeon/radeon_display.c linux-4.6.6/drivers/gpu/drm/radeon/radeon_display.c
  3836. --- linux-4.6.6.orig/drivers/gpu/drm/radeon/radeon_display.c 2016-08-10 12:54:57.000000000 +0200
  3837. +++ linux-4.6.6/drivers/gpu/drm/radeon/radeon_display.c 2016-08-22 21:27:46.929862072 +0200
  3838. @@ -1863,6 +1863,7 @@
  3839. struct radeon_device *rdev = dev->dev_private;
  3840. /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
  3841. + preempt_disable_rt();
  3842. /* Get optional system timestamp before query. */
  3843. if (stime)
  3844. @@ -1955,6 +1956,7 @@
  3845. *etime = ktime_get();
  3846. /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
  3847. + preempt_enable_rt();
  3848. /* Decode into vertical and horizontal scanout position. */
  3849. *vpos = position & 0x1fff;
  3850. diff -Nur linux-4.6.6.orig/drivers/i2c/busses/i2c-omap.c linux-4.6.6/drivers/i2c/busses/i2c-omap.c
  3851. --- linux-4.6.6.orig/drivers/i2c/busses/i2c-omap.c 2016-08-10 12:54:57.000000000 +0200
  3852. +++ linux-4.6.6/drivers/i2c/busses/i2c-omap.c 2016-08-22 21:27:46.945863099 +0200
  3853. @@ -995,15 +995,12 @@
  3854. u16 mask;
  3855. u16 stat;
  3856. - spin_lock(&omap->lock);
  3857. - mask = omap_i2c_read_reg(omap, OMAP_I2C_IE_REG);
  3858. stat = omap_i2c_read_reg(omap, OMAP_I2C_STAT_REG);
  3859. + mask = omap_i2c_read_reg(omap, OMAP_I2C_IE_REG);
  3860. if (stat & mask)
  3861. ret = IRQ_WAKE_THREAD;
  3862. - spin_unlock(&omap->lock);
  3863. -
  3864. return ret;
  3865. }
  3866. diff -Nur linux-4.6.6.orig/drivers/ide/alim15x3.c linux-4.6.6/drivers/ide/alim15x3.c
  3867. --- linux-4.6.6.orig/drivers/ide/alim15x3.c 2016-08-10 12:54:57.000000000 +0200
  3868. +++ linux-4.6.6/drivers/ide/alim15x3.c 2016-08-22 21:27:46.949863359 +0200
  3869. @@ -234,7 +234,7 @@
  3870. isa_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL);
  3871. - local_irq_save(flags);
  3872. + local_irq_save_nort(flags);
  3873. if (m5229_revision < 0xC2) {
  3874. /*
  3875. @@ -325,7 +325,7 @@
  3876. }
  3877. pci_dev_put(north);
  3878. pci_dev_put(isa_dev);
  3879. - local_irq_restore(flags);
  3880. + local_irq_restore_nort(flags);
  3881. return 0;
  3882. }
  3883. diff -Nur linux-4.6.6.orig/drivers/ide/hpt366.c linux-4.6.6/drivers/ide/hpt366.c
  3884. --- linux-4.6.6.orig/drivers/ide/hpt366.c 2016-08-10 12:54:57.000000000 +0200
  3885. +++ linux-4.6.6/drivers/ide/hpt366.c 2016-08-22 21:27:46.953863612 +0200
  3886. @@ -1236,7 +1236,7 @@
  3887. dma_old = inb(base + 2);
  3888. - local_irq_save(flags);
  3889. + local_irq_save_nort(flags);
  3890. dma_new = dma_old;
  3891. pci_read_config_byte(dev, hwif->channel ? 0x4b : 0x43, &masterdma);
  3892. @@ -1247,7 +1247,7 @@
  3893. if (dma_new != dma_old)
  3894. outb(dma_new, base + 2);
  3895. - local_irq_restore(flags);
  3896. + local_irq_restore_nort(flags);
  3897. printk(KERN_INFO " %s: BM-DMA at 0x%04lx-0x%04lx\n",
  3898. hwif->name, base, base + 7);
  3899. diff -Nur linux-4.6.6.orig/drivers/ide/ide-io.c linux-4.6.6/drivers/ide/ide-io.c
  3900. --- linux-4.6.6.orig/drivers/ide/ide-io.c 2016-08-10 12:54:57.000000000 +0200
  3901. +++ linux-4.6.6/drivers/ide/ide-io.c 2016-08-22 21:27:46.957863872 +0200
  3902. @@ -659,7 +659,7 @@
  3903. /* disable_irq_nosync ?? */
  3904. disable_irq(hwif->irq);
  3905. /* local CPU only, as if we were handling an interrupt */
  3906. - local_irq_disable();
  3907. + local_irq_disable_nort();
  3908. if (hwif->polling) {
  3909. startstop = handler(drive);
  3910. } else if (drive_is_ready(drive)) {
  3911. diff -Nur linux-4.6.6.orig/drivers/ide/ide-iops.c linux-4.6.6/drivers/ide/ide-iops.c
  3912. --- linux-4.6.6.orig/drivers/ide/ide-iops.c 2016-08-10 12:54:57.000000000 +0200
  3913. +++ linux-4.6.6/drivers/ide/ide-iops.c 2016-08-22 21:27:46.957863872 +0200
  3914. @@ -129,12 +129,12 @@
  3915. if ((stat & ATA_BUSY) == 0)
  3916. break;
  3917. - local_irq_restore(flags);
  3918. + local_irq_restore_nort(flags);
  3919. *rstat = stat;
  3920. return -EBUSY;
  3921. }
  3922. }
  3923. - local_irq_restore(flags);
  3924. + local_irq_restore_nort(flags);
  3925. }
  3926. /*
  3927. * Allow status to settle, then read it again.
  3928. diff -Nur linux-4.6.6.orig/drivers/ide/ide-io-std.c linux-4.6.6/drivers/ide/ide-io-std.c
  3929. --- linux-4.6.6.orig/drivers/ide/ide-io-std.c 2016-08-10 12:54:57.000000000 +0200
  3930. +++ linux-4.6.6/drivers/ide/ide-io-std.c 2016-08-22 21:27:46.957863872 +0200
  3931. @@ -175,7 +175,7 @@
  3932. unsigned long uninitialized_var(flags);
  3933. if ((io_32bit & 2) && !mmio) {
  3934. - local_irq_save(flags);
  3935. + local_irq_save_nort(flags);
  3936. ata_vlb_sync(io_ports->nsect_addr);
  3937. }
  3938. @@ -186,7 +186,7 @@
  3939. insl(data_addr, buf, words);
  3940. if ((io_32bit & 2) && !mmio)
  3941. - local_irq_restore(flags);
  3942. + local_irq_restore_nort(flags);
  3943. if (((len + 1) & 3) < 2)
  3944. return;
  3945. @@ -219,7 +219,7 @@
  3946. unsigned long uninitialized_var(flags);
  3947. if ((io_32bit & 2) && !mmio) {
  3948. - local_irq_save(flags);
  3949. + local_irq_save_nort(flags);
  3950. ata_vlb_sync(io_ports->nsect_addr);
  3951. }
  3952. @@ -230,7 +230,7 @@
  3953. outsl(data_addr, buf, words);
  3954. if ((io_32bit & 2) && !mmio)
  3955. - local_irq_restore(flags);
  3956. + local_irq_restore_nort(flags);
  3957. if (((len + 1) & 3) < 2)
  3958. return;
  3959. diff -Nur linux-4.6.6.orig/drivers/ide/ide-probe.c linux-4.6.6/drivers/ide/ide-probe.c
  3960. --- linux-4.6.6.orig/drivers/ide/ide-probe.c 2016-08-10 12:54:57.000000000 +0200
  3961. +++ linux-4.6.6/drivers/ide/ide-probe.c 2016-08-22 21:27:46.961864125 +0200
  3962. @@ -196,10 +196,10 @@
  3963. int bswap = 1;
  3964. /* local CPU only; some systems need this */
  3965. - local_irq_save(flags);
  3966. + local_irq_save_nort(flags);
  3967. /* read 512 bytes of id info */
  3968. hwif->tp_ops->input_data(drive, NULL, id, SECTOR_SIZE);
  3969. - local_irq_restore(flags);
  3970. + local_irq_restore_nort(flags);
  3971. drive->dev_flags |= IDE_DFLAG_ID_READ;
  3972. #ifdef DEBUG
  3973. diff -Nur linux-4.6.6.orig/drivers/ide/ide-taskfile.c linux-4.6.6/drivers/ide/ide-taskfile.c
  3974. --- linux-4.6.6.orig/drivers/ide/ide-taskfile.c 2016-08-10 12:54:57.000000000 +0200
  3975. +++ linux-4.6.6/drivers/ide/ide-taskfile.c 2016-08-22 21:27:46.961864125 +0200
  3976. @@ -250,7 +250,7 @@
  3977. page_is_high = PageHighMem(page);
  3978. if (page_is_high)
  3979. - local_irq_save(flags);
  3980. + local_irq_save_nort(flags);
  3981. buf = kmap_atomic(page) + offset;
  3982. @@ -271,7 +271,7 @@
  3983. kunmap_atomic(buf);
  3984. if (page_is_high)
  3985. - local_irq_restore(flags);
  3986. + local_irq_restore_nort(flags);
  3987. len -= nr_bytes;
  3988. }
  3989. @@ -414,7 +414,7 @@
  3990. }
  3991. if ((drive->dev_flags & IDE_DFLAG_UNMASK) == 0)
  3992. - local_irq_disable();
  3993. + local_irq_disable_nort();
  3994. ide_set_handler(drive, &task_pio_intr, WAIT_WORSTCASE);
  3995. diff -Nur linux-4.6.6.orig/drivers/infiniband/ulp/ipoib/ipoib_ib.c linux-4.6.6/drivers/infiniband/ulp/ipoib/ipoib_ib.c
  3996. --- linux-4.6.6.orig/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2016-08-10 12:54:57.000000000 +0200
  3997. +++ linux-4.6.6/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2016-08-22 21:27:46.969864638 +0200
  3998. @@ -51,8 +51,6 @@
  3999. "Enable data path debug tracing if > 0");
  4000. #endif
  4001. -static DEFINE_MUTEX(pkey_mutex);
  4002. -
  4003. struct ipoib_ah *ipoib_create_ah(struct net_device *dev,
  4004. struct ib_pd *pd, struct ib_ah_attr *attr)
  4005. {
  4006. diff -Nur linux-4.6.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.6.6/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
  4007. --- linux-4.6.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2016-08-10 12:54:57.000000000 +0200
  4008. +++ linux-4.6.6/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2016-08-22 21:27:46.969864638 +0200
  4009. @@ -883,7 +883,7 @@
  4010. ipoib_dbg_mcast(priv, "restarting multicast task\n");
  4011. - local_irq_save(flags);
  4012. + local_irq_save_nort(flags);
  4013. netif_addr_lock(dev);
  4014. spin_lock(&priv->lock);
  4015. @@ -965,7 +965,7 @@
  4016. spin_unlock(&priv->lock);
  4017. netif_addr_unlock(dev);
  4018. - local_irq_restore(flags);
  4019. + local_irq_restore_nort(flags);
  4020. /*
  4021. * make sure the in-flight joins have finished before we attempt
  4022. diff -Nur linux-4.6.6.orig/drivers/input/gameport/gameport.c linux-4.6.6/drivers/input/gameport/gameport.c
  4023. --- linux-4.6.6.orig/drivers/input/gameport/gameport.c 2016-08-10 12:54:57.000000000 +0200
  4024. +++ linux-4.6.6/drivers/input/gameport/gameport.c 2016-08-22 21:27:46.977865151 +0200
  4025. @@ -91,13 +91,13 @@
  4026. tx = ~0;
  4027. for (i = 0; i < 50; i++) {
  4028. - local_irq_save(flags);
  4029. + local_irq_save_nort(flags);
  4030. t1 = ktime_get_ns();
  4031. for (t = 0; t < 50; t++)
  4032. gameport_read(gameport);
  4033. t2 = ktime_get_ns();
  4034. t3 = ktime_get_ns();
  4035. - local_irq_restore(flags);
  4036. + local_irq_restore_nort(flags);
  4037. udelay(i * 10);
  4038. t = (t2 - t1) - (t3 - t2);
  4039. if (t < tx)
  4040. @@ -124,12 +124,12 @@
  4041. tx = 1 << 30;
  4042. for(i = 0; i < 50; i++) {
  4043. - local_irq_save(flags);
  4044. + local_irq_save_nort(flags);
  4045. GET_TIME(t1);
  4046. for (t = 0; t < 50; t++) gameport_read(gameport);
  4047. GET_TIME(t2);
  4048. GET_TIME(t3);
  4049. - local_irq_restore(flags);
  4050. + local_irq_restore_nort(flags);
  4051. udelay(i * 10);
  4052. if ((t = DELTA(t2,t1) - DELTA(t3,t2)) < tx) tx = t;
  4053. }
  4054. @@ -148,11 +148,11 @@
  4055. tx = 1 << 30;
  4056. for(i = 0; i < 50; i++) {
  4057. - local_irq_save(flags);
  4058. + local_irq_save_nort(flags);
  4059. t1 = rdtsc();
  4060. for (t = 0; t < 50; t++) gameport_read(gameport);
  4061. t2 = rdtsc();
  4062. - local_irq_restore(flags);
  4063. + local_irq_restore_nort(flags);
  4064. udelay(i * 10);
  4065. if (t2 - t1 < tx) tx = t2 - t1;
  4066. }
  4067. diff -Nur linux-4.6.6.orig/drivers/iommu/amd_iommu.c linux-4.6.6/drivers/iommu/amd_iommu.c
  4068. --- linux-4.6.6.orig/drivers/iommu/amd_iommu.c 2016-08-10 12:54:57.000000000 +0200
  4069. +++ linux-4.6.6/drivers/iommu/amd_iommu.c 2016-08-22 21:27:46.997866437 +0200
  4070. @@ -2165,10 +2165,10 @@
  4071. int ret;
  4072. /*
  4073. - * Must be called with IRQs disabled. Warn here to detect early
  4074. - * when its not.
  4075. + * Must be called with IRQs disabled on a non RT kernel. Warn here to
  4076. + * detect early when its not.
  4077. */
  4078. - WARN_ON(!irqs_disabled());
  4079. + WARN_ON_NONRT(!irqs_disabled());
  4080. /* lock domain */
  4081. spin_lock(&domain->lock);
  4082. @@ -2331,10 +2331,10 @@
  4083. struct protection_domain *domain;
  4084. /*
  4085. - * Must be called with IRQs disabled. Warn here to detect early
  4086. - * when its not.
  4087. + * Must be called with IRQs disabled on a non RT kernel. Warn here to
  4088. + * detect early when its not.
  4089. */
  4090. - WARN_ON(!irqs_disabled());
  4091. + WARN_ON_NONRT(!irqs_disabled());
  4092. if (WARN_ON(!dev_data->domain))
  4093. return;
  4094. diff -Nur linux-4.6.6.orig/drivers/leds/trigger/Kconfig linux-4.6.6/drivers/leds/trigger/Kconfig
  4095. --- linux-4.6.6.orig/drivers/leds/trigger/Kconfig 2016-08-10 12:54:57.000000000 +0200
  4096. +++ linux-4.6.6/drivers/leds/trigger/Kconfig 2016-08-22 21:27:47.005866951 +0200
  4097. @@ -61,7 +61,7 @@
  4098. config LEDS_TRIGGER_CPU
  4099. bool "LED CPU Trigger"
  4100. - depends on LEDS_TRIGGERS
  4101. + depends on LEDS_TRIGGERS && !PREEMPT_RT_BASE
  4102. help
  4103. This allows LEDs to be controlled by active CPUs. This shows
  4104. the active CPUs across an array of LEDs so you can see which
  4105. diff -Nur linux-4.6.6.orig/drivers/md/bcache/Kconfig linux-4.6.6/drivers/md/bcache/Kconfig
  4106. --- linux-4.6.6.orig/drivers/md/bcache/Kconfig 2016-08-10 12:54:57.000000000 +0200
  4107. +++ linux-4.6.6/drivers/md/bcache/Kconfig 2016-08-22 21:27:47.009867204 +0200
  4108. @@ -1,6 +1,7 @@
  4109. config BCACHE
  4110. tristate "Block device as cache"
  4111. + depends on !PREEMPT_RT_FULL
  4112. ---help---
  4113. Allows a block device to be used as cache for other devices; uses
  4114. a btree for indexing and the layout is optimized for SSDs.
  4115. diff -Nur linux-4.6.6.orig/drivers/md/dm.c linux-4.6.6/drivers/md/dm.c
  4116. --- linux-4.6.6.orig/drivers/md/dm.c 2016-08-10 12:54:57.000000000 +0200
  4117. +++ linux-4.6.6/drivers/md/dm.c 2016-08-22 21:27:47.025868232 +0200
  4118. @@ -2187,7 +2187,7 @@
  4119. /* Establish tio->ti before queuing work (map_tio_request) */
  4120. tio->ti = ti;
  4121. queue_kthread_work(&md->kworker, &tio->work);
  4122. - BUG_ON(!irqs_disabled());
  4123. + BUG_ON_NONRT(!irqs_disabled());
  4124. }
  4125. }
  4126. diff -Nur linux-4.6.6.orig/drivers/md/raid5.c linux-4.6.6/drivers/md/raid5.c
  4127. --- linux-4.6.6.orig/drivers/md/raid5.c 2016-08-10 12:54:57.000000000 +0200
  4128. +++ linux-4.6.6/drivers/md/raid5.c 2016-08-22 21:27:47.041869254 +0200
  4129. @@ -1918,8 +1918,9 @@
  4130. struct raid5_percpu *percpu;
  4131. unsigned long cpu;
  4132. - cpu = get_cpu();
  4133. + cpu = get_cpu_light();
  4134. percpu = per_cpu_ptr(conf->percpu, cpu);
  4135. + spin_lock(&percpu->lock);
  4136. if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
  4137. ops_run_biofill(sh);
  4138. overlap_clear++;
  4139. @@ -1975,7 +1976,8 @@
  4140. if (test_and_clear_bit(R5_Overlap, &dev->flags))
  4141. wake_up(&sh->raid_conf->wait_for_overlap);
  4142. }
  4143. - put_cpu();
  4144. + spin_unlock(&percpu->lock);
  4145. + put_cpu_light();
  4146. }
  4147. static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp)
  4148. @@ -6415,6 +6417,7 @@
  4149. __func__, cpu);
  4150. break;
  4151. }
  4152. + spin_lock_init(&per_cpu_ptr(conf->percpu, cpu)->lock);
  4153. }
  4154. put_online_cpus();
  4155. diff -Nur linux-4.6.6.orig/drivers/md/raid5.h linux-4.6.6/drivers/md/raid5.h
  4156. --- linux-4.6.6.orig/drivers/md/raid5.h 2016-08-10 12:54:57.000000000 +0200
  4157. +++ linux-4.6.6/drivers/md/raid5.h 2016-08-22 21:27:47.041869254 +0200
  4158. @@ -504,6 +504,7 @@
  4159. int recovery_disabled;
  4160. /* per cpu variables */
  4161. struct raid5_percpu {
  4162. + spinlock_t lock; /* Protection for -RT */
  4163. struct page *spare_page; /* Used when checking P/Q in raid6 */
  4164. struct flex_array *scribble; /* space for constructing buffer
  4165. * lists and performing address
  4166. diff -Nur linux-4.6.6.orig/drivers/misc/hwlat_detector.c linux-4.6.6/drivers/misc/hwlat_detector.c
  4167. --- linux-4.6.6.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100
  4168. +++ linux-4.6.6/drivers/misc/hwlat_detector.c 2016-08-22 21:27:47.053870028 +0200
  4169. @@ -0,0 +1,1240 @@
  4170. +/*
  4171. + * hwlat_detector.c - A simple Hardware Latency detector.
  4172. + *
  4173. + * Use this module to detect large system latencies induced by the behavior of
  4174. + * certain underlying system hardware or firmware, independent of Linux itself.
  4175. + * The code was developed originally to detect the presence of SMIs on Intel
  4176. + * and AMD systems, although there is no dependency upon x86 herein.
  4177. + *
  4178. + * The classical example usage of this module is in detecting the presence of
  4179. + * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a
  4180. + * somewhat special form of hardware interrupt spawned from earlier CPU debug
  4181. + * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge
  4182. + * LPC (or other device) to generate a special interrupt under certain
  4183. + * circumstances, for example, upon expiration of a special SMI timer device,
  4184. + * due to certain external thermal readings, on certain I/O address accesses,
  4185. + * and other situations. An SMI hits a special CPU pin, triggers a special
  4186. + * SMI mode (complete with special memory map), and the OS is unaware.
  4187. + *
  4188. + * Although certain hardware-inducing latencies are necessary (for example,
  4189. + * a modern system often requires an SMI handler for correct thermal control
  4190. + * and remote management) they can wreak havoc upon any OS-level performance
  4191. + * guarantees toward low-latency, especially when the OS is not even made
  4192. + * aware of the presence of these interrupts. For this reason, we need a
  4193. + * somewhat brute force mechanism to detect these interrupts. In this case,
  4194. + * we do it by hogging all of the CPU(s) for configurable timer intervals,
  4195. + * sampling the built-in CPU timer, looking for discontiguous readings.
  4196. + *
  4197. + * WARNING: This implementation necessarily introduces latencies. Therefore,
  4198. + * you should NEVER use this module in a production environment
  4199. + * requiring any kind of low-latency performance guarantee(s).
  4200. + *
  4201. + * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
  4202. + *
  4203. + * Includes useful feedback from Clark Williams <clark@redhat.com>
  4204. + *
  4205. + * This file is licensed under the terms of the GNU General Public
  4206. + * License version 2. This program is licensed "as is" without any
  4207. + * warranty of any kind, whether express or implied.
  4208. + */
  4209. +
  4210. +#include <linux/module.h>
  4211. +#include <linux/init.h>
  4212. +#include <linux/ring_buffer.h>
  4213. +#include <linux/time.h>
  4214. +#include <linux/hrtimer.h>
  4215. +#include <linux/kthread.h>
  4216. +#include <linux/debugfs.h>
  4217. +#include <linux/seq_file.h>
  4218. +#include <linux/uaccess.h>
  4219. +#include <linux/version.h>
  4220. +#include <linux/delay.h>
  4221. +#include <linux/slab.h>
  4222. +#include <linux/trace_clock.h>
  4223. +
  4224. +#define BUF_SIZE_DEFAULT 262144UL /* 8K*(sizeof(entry)) */
  4225. +#define BUF_FLAGS (RB_FL_OVERWRITE) /* no block on full */
  4226. +#define U64STR_SIZE 22 /* 20 digits max */
  4227. +
  4228. +#define VERSION "1.0.0"
  4229. +#define BANNER "hwlat_detector: "
  4230. +#define DRVNAME "hwlat_detector"
  4231. +#define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */
  4232. +#define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */
  4233. +#define DEFAULT_LAT_THRESHOLD 10 /* 10us */
  4234. +
  4235. +/* Module metadata */
  4236. +
  4237. +MODULE_LICENSE("GPL");
  4238. +MODULE_AUTHOR("Jon Masters <jcm@redhat.com>");
  4239. +MODULE_DESCRIPTION("A simple hardware latency detector");
  4240. +MODULE_VERSION(VERSION);
  4241. +
  4242. +/* Module parameters */
  4243. +
  4244. +static int debug;
  4245. +static int enabled;
  4246. +static int threshold;
  4247. +
  4248. +module_param(debug, int, 0); /* enable debug */
  4249. +module_param(enabled, int, 0); /* enable detector */
  4250. +module_param(threshold, int, 0); /* latency threshold */
  4251. +
  4252. +/* Buffering and sampling */
  4253. +
  4254. +static struct ring_buffer *ring_buffer; /* sample buffer */
  4255. +static DEFINE_MUTEX(ring_buffer_mutex); /* lock changes */
  4256. +static unsigned long buf_size = BUF_SIZE_DEFAULT;
  4257. +static struct task_struct *kthread; /* sampling thread */
  4258. +
  4259. +/* DebugFS filesystem entries */
  4260. +
  4261. +static struct dentry *debug_dir; /* debugfs directory */
  4262. +static struct dentry *debug_max; /* maximum TSC delta */
  4263. +static struct dentry *debug_count; /* total detect count */
  4264. +static struct dentry *debug_sample_width; /* sample width us */
  4265. +static struct dentry *debug_sample_window; /* sample window us */
  4266. +static struct dentry *debug_sample; /* raw samples us */
  4267. +static struct dentry *debug_threshold; /* threshold us */
  4268. +static struct dentry *debug_enable; /* enable/disable */
  4269. +
  4270. +/* Individual samples and global state */
  4271. +
  4272. +struct sample; /* latency sample */
  4273. +struct data; /* Global state */
  4274. +
  4275. +/* Sampling functions */
  4276. +static int __buffer_add_sample(struct sample *sample);
  4277. +static struct sample *buffer_get_sample(struct sample *sample);
  4278. +
  4279. +/* Threading and state */
  4280. +static int kthread_fn(void *unused);
  4281. +static int start_kthread(void);
  4282. +static int stop_kthread(void);
  4283. +static void __reset_stats(void);
  4284. +static int init_stats(void);
  4285. +
  4286. +/* Debugfs interface */
  4287. +static ssize_t simple_data_read(struct file *filp, char __user *ubuf,
  4288. + size_t cnt, loff_t *ppos, const u64 *entry);
  4289. +static ssize_t simple_data_write(struct file *filp, const char __user *ubuf,
  4290. + size_t cnt, loff_t *ppos, u64 *entry);
  4291. +static int debug_sample_fopen(struct inode *inode, struct file *filp);
  4292. +static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf,
  4293. + size_t cnt, loff_t *ppos);
  4294. +static int debug_sample_release(struct inode *inode, struct file *filp);
  4295. +static int debug_enable_fopen(struct inode *inode, struct file *filp);
  4296. +static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf,
  4297. + size_t cnt, loff_t *ppos);
  4298. +static ssize_t debug_enable_fwrite(struct file *file,
  4299. + const char __user *user_buffer,
  4300. + size_t user_size, loff_t *offset);
  4301. +
  4302. +/* Initialization functions */
  4303. +static int init_debugfs(void);
  4304. +static void free_debugfs(void);
  4305. +static int detector_init(void);
  4306. +static void detector_exit(void);
  4307. +
  4308. +/* Individual latency samples are stored here when detected and packed into
  4309. + * the ring_buffer circular buffer, where they are overwritten when
  4310. + * more than buf_size/sizeof(sample) samples are received. */
  4311. +struct sample {
  4312. + u64 seqnum; /* unique sequence */
  4313. + u64 duration; /* ktime delta */
  4314. + u64 outer_duration; /* ktime delta (outer loop) */
  4315. + struct timespec timestamp; /* wall time */
  4316. + unsigned long lost;
  4317. +};
  4318. +
  4319. +/* keep the global state somewhere. */
  4320. +static struct data {
  4321. +
  4322. + struct mutex lock; /* protect changes */
  4323. +
  4324. + u64 count; /* total since reset */
  4325. + u64 max_sample; /* max hardware latency */
  4326. + u64 threshold; /* sample threshold level */
  4327. +
  4328. + u64 sample_window; /* total sampling window (on+off) */
  4329. + u64 sample_width; /* active sampling portion of window */
  4330. +
  4331. + atomic_t sample_open; /* whether the sample file is open */
  4332. +
  4333. + wait_queue_head_t wq; /* waitqeue for new sample values */
  4334. +
  4335. +} data;
  4336. +
  4337. +/**
  4338. + * __buffer_add_sample - add a new latency sample recording to the ring buffer
  4339. + * @sample: The new latency sample value
  4340. + *
  4341. + * This receives a new latency sample and records it in a global ring buffer.
  4342. + * No additional locking is used in this case.
  4343. + */
  4344. +static int __buffer_add_sample(struct sample *sample)
  4345. +{
  4346. + return ring_buffer_write(ring_buffer,
  4347. + sizeof(struct sample), sample);
  4348. +}
  4349. +
  4350. +/**
  4351. + * buffer_get_sample - remove a hardware latency sample from the ring buffer
  4352. + * @sample: Pre-allocated storage for the sample
  4353. + *
  4354. + * This retrieves a hardware latency sample from the global circular buffer
  4355. + */
  4356. +static struct sample *buffer_get_sample(struct sample *sample)
  4357. +{
  4358. + struct ring_buffer_event *e = NULL;
  4359. + struct sample *s = NULL;
  4360. + unsigned int cpu = 0;
  4361. +
  4362. + if (!sample)
  4363. + return NULL;
  4364. +
  4365. + mutex_lock(&ring_buffer_mutex);
  4366. + for_each_online_cpu(cpu) {
  4367. + e = ring_buffer_consume(ring_buffer, cpu, NULL, &sample->lost);
  4368. + if (e)
  4369. + break;
  4370. + }
  4371. +
  4372. + if (e) {
  4373. + s = ring_buffer_event_data(e);
  4374. + memcpy(sample, s, sizeof(struct sample));
  4375. + } else
  4376. + sample = NULL;
  4377. + mutex_unlock(&ring_buffer_mutex);
  4378. +
  4379. + return sample;
  4380. +}
  4381. +
  4382. +#ifndef CONFIG_TRACING
  4383. +#define time_type ktime_t
  4384. +#define time_get() ktime_get()
  4385. +#define time_to_us(x) ktime_to_us(x)
  4386. +#define time_sub(a, b) ktime_sub(a, b)
  4387. +#define init_time(a, b) (a).tv64 = b
  4388. +#define time_u64(a) ((a).tv64)
  4389. +#else
  4390. +#define time_type u64
  4391. +#define time_get() trace_clock_local()
  4392. +#define time_to_us(x) div_u64(x, 1000)
  4393. +#define time_sub(a, b) ((a) - (b))
  4394. +#define init_time(a, b) (a = b)
  4395. +#define time_u64(a) a
  4396. +#endif
  4397. +/**
  4398. + * get_sample - sample the CPU TSC and look for likely hardware latencies
  4399. + *
  4400. + * Used to repeatedly capture the CPU TSC (or similar), looking for potential
  4401. + * hardware-induced latency. Called with interrupts disabled and with
  4402. + * data.lock held.
  4403. + */
  4404. +static int get_sample(void)
  4405. +{
  4406. + time_type start, t1, t2, last_t2;
  4407. + s64 diff, total = 0;
  4408. + u64 sample = 0;
  4409. + u64 outer_sample = 0;
  4410. + int ret = -1;
  4411. +
  4412. + init_time(last_t2, 0);
  4413. + start = time_get(); /* start timestamp */
  4414. +
  4415. + do {
  4416. +
  4417. + t1 = time_get(); /* we'll look for a discontinuity */
  4418. + t2 = time_get();
  4419. +
  4420. + if (time_u64(last_t2)) {
  4421. + /* Check the delta from outer loop (t2 to next t1) */
  4422. + diff = time_to_us(time_sub(t1, last_t2));
  4423. + /* This shouldn't happen */
  4424. + if (diff < 0) {
  4425. + pr_err(BANNER "time running backwards\n");
  4426. + goto out;
  4427. + }
  4428. + if (diff > outer_sample)
  4429. + outer_sample = diff;
  4430. + }
  4431. + last_t2 = t2;
  4432. +
  4433. + total = time_to_us(time_sub(t2, start)); /* sample width */
  4434. +
  4435. + /* This checks the inner loop (t1 to t2) */
  4436. + diff = time_to_us(time_sub(t2, t1)); /* current diff */
  4437. +
  4438. + /* This shouldn't happen */
  4439. + if (diff < 0) {
  4440. + pr_err(BANNER "time running backwards\n");
  4441. + goto out;
  4442. + }
  4443. +
  4444. + if (diff > sample)
  4445. + sample = diff; /* only want highest value */
  4446. +
  4447. + } while (total <= data.sample_width);
  4448. +
  4449. + ret = 0;
  4450. +
  4451. + /* If we exceed the threshold value, we have found a hardware latency */
  4452. + if (sample > data.threshold || outer_sample > data.threshold) {
  4453. + struct sample s;
  4454. +
  4455. + ret = 1;
  4456. +
  4457. + data.count++;
  4458. + s.seqnum = data.count;
  4459. + s.duration = sample;
  4460. + s.outer_duration = outer_sample;
  4461. + s.timestamp = CURRENT_TIME;
  4462. + __buffer_add_sample(&s);
  4463. +
  4464. + /* Keep a running maximum ever recorded hardware latency */
  4465. + if (sample > data.max_sample)
  4466. + data.max_sample = sample;
  4467. + }
  4468. +
  4469. +out:
  4470. + return ret;
  4471. +}
  4472. +
  4473. +/*
  4474. + * kthread_fn - The CPU time sampling/hardware latency detection kernel thread
  4475. + * @unused: A required part of the kthread API.
  4476. + *
  4477. + * Used to periodically sample the CPU TSC via a call to get_sample. We
  4478. + * disable interrupts, which does (intentionally) introduce latency since we
  4479. + * need to ensure nothing else might be running (and thus pre-empting).
  4480. + * Obviously this should never be used in production environments.
  4481. + *
  4482. + * Currently this runs on which ever CPU it was scheduled on, but most
  4483. + * real-worald hardware latency situations occur across several CPUs,
  4484. + * but we might later generalize this if we find there are any actualy
  4485. + * systems with alternate SMI delivery or other hardware latencies.
  4486. + */
  4487. +static int kthread_fn(void *unused)
  4488. +{
  4489. + int ret;
  4490. + u64 interval;
  4491. +
  4492. + while (!kthread_should_stop()) {
  4493. +
  4494. + mutex_lock(&data.lock);
  4495. +
  4496. + local_irq_disable();
  4497. + ret = get_sample();
  4498. + local_irq_enable();
  4499. +
  4500. + if (ret > 0)
  4501. + wake_up(&data.wq); /* wake up reader(s) */
  4502. +
  4503. + interval = data.sample_window - data.sample_width;
  4504. + do_div(interval, USEC_PER_MSEC); /* modifies interval value */
  4505. +
  4506. + mutex_unlock(&data.lock);
  4507. +
  4508. + if (msleep_interruptible(interval))
  4509. + break;
  4510. + }
  4511. +
  4512. + return 0;
  4513. +}
  4514. +
  4515. +/**
  4516. + * start_kthread - Kick off the hardware latency sampling/detector kthread
  4517. + *
  4518. + * This starts a kernel thread that will sit and sample the CPU timestamp
  4519. + * counter (TSC or similar) and look for potential hardware latencies.
  4520. + */
  4521. +static int start_kthread(void)
  4522. +{
  4523. + kthread = kthread_run(kthread_fn, NULL,
  4524. + DRVNAME);
  4525. + if (IS_ERR(kthread)) {
  4526. + pr_err(BANNER "could not start sampling thread\n");
  4527. + enabled = 0;
  4528. + return -ENOMEM;
  4529. + }
  4530. +
  4531. + return 0;
  4532. +}
  4533. +
  4534. +/**
  4535. + * stop_kthread - Inform the hardware latency samping/detector kthread to stop
  4536. + *
  4537. + * This kicks the running hardware latency sampling/detector kernel thread and
  4538. + * tells it to stop sampling now. Use this on unload and at system shutdown.
  4539. + */
  4540. +static int stop_kthread(void)
  4541. +{
  4542. + int ret;
  4543. +
  4544. + ret = kthread_stop(kthread);
  4545. +
  4546. + return ret;
  4547. +}
  4548. +
  4549. +/**
  4550. + * __reset_stats - Reset statistics for the hardware latency detector
  4551. + *
  4552. + * We use data to store various statistics and global state. We call this
  4553. + * function in order to reset those when "enable" is toggled on or off, and
  4554. + * also at initialization. Should be called with data.lock held.
  4555. + */
  4556. +static void __reset_stats(void)
  4557. +{
  4558. + data.count = 0;
  4559. + data.max_sample = 0;
  4560. + ring_buffer_reset(ring_buffer); /* flush out old sample entries */
  4561. +}
  4562. +
  4563. +/**
  4564. + * init_stats - Setup global state statistics for the hardware latency detector
  4565. + *
  4566. + * We use data to store various statistics and global state. We also use
  4567. + * a global ring buffer (ring_buffer) to keep raw samples of detected hardware
  4568. + * induced system latencies. This function initializes these structures and
  4569. + * allocates the global ring buffer also.
  4570. + */
  4571. +static int init_stats(void)
  4572. +{
  4573. + int ret = -ENOMEM;
  4574. +
  4575. + mutex_init(&data.lock);
  4576. + init_waitqueue_head(&data.wq);
  4577. + atomic_set(&data.sample_open, 0);
  4578. +
  4579. + ring_buffer = ring_buffer_alloc(buf_size, BUF_FLAGS);
  4580. +
  4581. + if (WARN(!ring_buffer, KERN_ERR BANNER
  4582. + "failed to allocate ring buffer!\n"))
  4583. + goto out;
  4584. +
  4585. + __reset_stats();
  4586. + data.threshold = threshold ?: DEFAULT_LAT_THRESHOLD; /* threshold us */
  4587. + data.sample_window = DEFAULT_SAMPLE_WINDOW; /* window us */
  4588. + data.sample_width = DEFAULT_SAMPLE_WIDTH; /* width us */
  4589. +
  4590. + ret = 0;
  4591. +
  4592. +out:
  4593. + return ret;
  4594. +
  4595. +}
  4596. +
  4597. +/*
  4598. + * simple_data_read - Wrapper read function for global state debugfs entries
  4599. + * @filp: The active open file structure for the debugfs "file"
  4600. + * @ubuf: The userspace provided buffer to read value into
  4601. + * @cnt: The maximum number of bytes to read
  4602. + * @ppos: The current "file" position
  4603. + * @entry: The entry to read from
  4604. + *
  4605. + * This function provides a generic read implementation for the global state
  4606. + * "data" structure debugfs filesystem entries. It would be nice to use
  4607. + * simple_attr_read directly, but we need to make sure that the data.lock
  4608. + * is held during the actual read.
  4609. + */
  4610. +static ssize_t simple_data_read(struct file *filp, char __user *ubuf,
  4611. + size_t cnt, loff_t *ppos, const u64 *entry)
  4612. +{
  4613. + char buf[U64STR_SIZE];
  4614. + u64 val = 0;
  4615. + int len = 0;
  4616. +
  4617. + memset(buf, 0, sizeof(buf));
  4618. +
  4619. + if (!entry)
  4620. + return -EFAULT;
  4621. +
  4622. + mutex_lock(&data.lock);
  4623. + val = *entry;
  4624. + mutex_unlock(&data.lock);
  4625. +
  4626. + len = snprintf(buf, sizeof(buf), "%llu\n", (unsigned long long)val);
  4627. +
  4628. + return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
  4629. +
  4630. +}
  4631. +
  4632. +/*
  4633. + * simple_data_write - Wrapper write function for global state debugfs entries
  4634. + * @filp: The active open file structure for the debugfs "file"
  4635. + * @ubuf: The userspace provided buffer to write value from
  4636. + * @cnt: The maximum number of bytes to write
  4637. + * @ppos: The current "file" position
  4638. + * @entry: The entry to write to
  4639. + *
  4640. + * This function provides a generic write implementation for the global state
  4641. + * "data" structure debugfs filesystem entries. It would be nice to use
  4642. + * simple_attr_write directly, but we need to make sure that the data.lock
  4643. + * is held during the actual write.
  4644. + */
  4645. +static ssize_t simple_data_write(struct file *filp, const char __user *ubuf,
  4646. + size_t cnt, loff_t *ppos, u64 *entry)
  4647. +{
  4648. + char buf[U64STR_SIZE];
  4649. + int csize = min(cnt, sizeof(buf));
  4650. + u64 val = 0;
  4651. + int err = 0;
  4652. +
  4653. + memset(buf, '\0', sizeof(buf));
  4654. + if (copy_from_user(buf, ubuf, csize))
  4655. + return -EFAULT;
  4656. +
  4657. + buf[U64STR_SIZE-1] = '\0'; /* just in case */
  4658. + err = kstrtoull(buf, 10, &val);
  4659. + if (err)
  4660. + return -EINVAL;
  4661. +
  4662. + mutex_lock(&data.lock);
  4663. + *entry = val;
  4664. + mutex_unlock(&data.lock);
  4665. +
  4666. + return csize;
  4667. +}
  4668. +
  4669. +/**
  4670. + * debug_count_fopen - Open function for "count" debugfs entry
  4671. + * @inode: The in-kernel inode representation of the debugfs "file"
  4672. + * @filp: The active open file structure for the debugfs "file"
  4673. + *
  4674. + * This function provides an open implementation for the "count" debugfs
  4675. + * interface to the hardware latency detector.
  4676. + */
  4677. +static int debug_count_fopen(struct inode *inode, struct file *filp)
  4678. +{
  4679. + return 0;
  4680. +}
  4681. +
  4682. +/**
  4683. + * debug_count_fread - Read function for "count" debugfs entry
  4684. + * @filp: The active open file structure for the debugfs "file"
  4685. + * @ubuf: The userspace provided buffer to read value into
  4686. + * @cnt: The maximum number of bytes to read
  4687. + * @ppos: The current "file" position
  4688. + *
  4689. + * This function provides a read implementation for the "count" debugfs
  4690. + * interface to the hardware latency detector. Can be used to read the
  4691. + * number of latency readings exceeding the configured threshold since
  4692. + * the detector was last reset (e.g. by writing a zero into "count").
  4693. + */
  4694. +static ssize_t debug_count_fread(struct file *filp, char __user *ubuf,
  4695. + size_t cnt, loff_t *ppos)
  4696. +{
  4697. + return simple_data_read(filp, ubuf, cnt, ppos, &data.count);
  4698. +}
  4699. +
  4700. +/**
  4701. + * debug_count_fwrite - Write function for "count" debugfs entry
  4702. + * @filp: The active open file structure for the debugfs "file"
  4703. + * @ubuf: The user buffer that contains the value to write
  4704. + * @cnt: The maximum number of bytes to write to "file"
  4705. + * @ppos: The current position in the debugfs "file"
  4706. + *
  4707. + * This function provides a write implementation for the "count" debugfs
  4708. + * interface to the hardware latency detector. Can be used to write a
  4709. + * desired value, especially to zero the total count.
  4710. + */
  4711. +static ssize_t debug_count_fwrite(struct file *filp,
  4712. + const char __user *ubuf,
  4713. + size_t cnt,
  4714. + loff_t *ppos)
  4715. +{
  4716. + return simple_data_write(filp, ubuf, cnt, ppos, &data.count);
  4717. +}
  4718. +
  4719. +/**
  4720. + * debug_enable_fopen - Dummy open function for "enable" debugfs interface
  4721. + * @inode: The in-kernel inode representation of the debugfs "file"
  4722. + * @filp: The active open file structure for the debugfs "file"
  4723. + *
  4724. + * This function provides an open implementation for the "enable" debugfs
  4725. + * interface to the hardware latency detector.
  4726. + */
  4727. +static int debug_enable_fopen(struct inode *inode, struct file *filp)
  4728. +{
  4729. + return 0;
  4730. +}
  4731. +
  4732. +/**
  4733. + * debug_enable_fread - Read function for "enable" debugfs interface
  4734. + * @filp: The active open file structure for the debugfs "file"
  4735. + * @ubuf: The userspace provided buffer to read value into
  4736. + * @cnt: The maximum number of bytes to read
  4737. + * @ppos: The current "file" position
  4738. + *
  4739. + * This function provides a read implementation for the "enable" debugfs
  4740. + * interface to the hardware latency detector. Can be used to determine
  4741. + * whether the detector is currently enabled ("0\n" or "1\n" returned).
  4742. + */
  4743. +static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf,
  4744. + size_t cnt, loff_t *ppos)
  4745. +{
  4746. + char buf[4];
  4747. +
  4748. + if ((cnt < sizeof(buf)) || (*ppos))
  4749. + return 0;
  4750. +
  4751. + buf[0] = enabled ? '1' : '0';
  4752. + buf[1] = '\n';
  4753. + buf[2] = '\0';
  4754. + if (copy_to_user(ubuf, buf, strlen(buf)))
  4755. + return -EFAULT;
  4756. + return *ppos = strlen(buf);
  4757. +}
  4758. +
  4759. +/**
  4760. + * debug_enable_fwrite - Write function for "enable" debugfs interface
  4761. + * @filp: The active open file structure for the debugfs "file"
  4762. + * @ubuf: The user buffer that contains the value to write
  4763. + * @cnt: The maximum number of bytes to write to "file"
  4764. + * @ppos: The current position in the debugfs "file"
  4765. + *
  4766. + * This function provides a write implementation for the "enable" debugfs
  4767. + * interface to the hardware latency detector. Can be used to enable or
  4768. + * disable the detector, which will have the side-effect of possibly
  4769. + * also resetting the global stats and kicking off the measuring
  4770. + * kthread (on an enable) or the converse (upon a disable).
  4771. + */
  4772. +static ssize_t debug_enable_fwrite(struct file *filp,
  4773. + const char __user *ubuf,
  4774. + size_t cnt,
  4775. + loff_t *ppos)
  4776. +{
  4777. + char buf[4];
  4778. + int csize = min(cnt, sizeof(buf));
  4779. + long val = 0;
  4780. + int err = 0;
  4781. +
  4782. + memset(buf, '\0', sizeof(buf));
  4783. + if (copy_from_user(buf, ubuf, csize))
  4784. + return -EFAULT;
  4785. +
  4786. + buf[sizeof(buf)-1] = '\0'; /* just in case */
  4787. + err = kstrtoul(buf, 10, &val);
  4788. + if (err)
  4789. + return -EINVAL;
  4790. +
  4791. + if (val) {
  4792. + if (enabled)
  4793. + goto unlock;
  4794. + enabled = 1;
  4795. + __reset_stats();
  4796. + if (start_kthread())
  4797. + return -EFAULT;
  4798. + } else {
  4799. + if (!enabled)
  4800. + goto unlock;
  4801. + enabled = 0;
  4802. + err = stop_kthread();
  4803. + if (err) {
  4804. + pr_err(BANNER "cannot stop kthread\n");
  4805. + return -EFAULT;
  4806. + }
  4807. + wake_up(&data.wq); /* reader(s) should return */
  4808. + }
  4809. +unlock:
  4810. + return csize;
  4811. +}
  4812. +
  4813. +/**
  4814. + * debug_max_fopen - Open function for "max" debugfs entry
  4815. + * @inode: The in-kernel inode representation of the debugfs "file"
  4816. + * @filp: The active open file structure for the debugfs "file"
  4817. + *
  4818. + * This function provides an open implementation for the "max" debugfs
  4819. + * interface to the hardware latency detector.
  4820. + */
  4821. +static int debug_max_fopen(struct inode *inode, struct file *filp)
  4822. +{
  4823. + return 0;
  4824. +}
  4825. +
  4826. +/**
  4827. + * debug_max_fread - Read function for "max" debugfs entry
  4828. + * @filp: The active open file structure for the debugfs "file"
  4829. + * @ubuf: The userspace provided buffer to read value into
  4830. + * @cnt: The maximum number of bytes to read
  4831. + * @ppos: The current "file" position
  4832. + *
  4833. + * This function provides a read implementation for the "max" debugfs
  4834. + * interface to the hardware latency detector. Can be used to determine
  4835. + * the maximum latency value observed since it was last reset.
  4836. + */
  4837. +static ssize_t debug_max_fread(struct file *filp, char __user *ubuf,
  4838. + size_t cnt, loff_t *ppos)
  4839. +{
  4840. + return simple_data_read(filp, ubuf, cnt, ppos, &data.max_sample);
  4841. +}
  4842. +
  4843. +/**
  4844. + * debug_max_fwrite - Write function for "max" debugfs entry
  4845. + * @filp: The active open file structure for the debugfs "file"
  4846. + * @ubuf: The user buffer that contains the value to write
  4847. + * @cnt: The maximum number of bytes to write to "file"
  4848. + * @ppos: The current position in the debugfs "file"
  4849. + *
  4850. + * This function provides a write implementation for the "max" debugfs
  4851. + * interface to the hardware latency detector. Can be used to reset the
  4852. + * maximum or set it to some other desired value - if, then, subsequent
  4853. + * measurements exceed this value, the maximum will be updated.
  4854. + */
  4855. +static ssize_t debug_max_fwrite(struct file *filp,
  4856. + const char __user *ubuf,
  4857. + size_t cnt,
  4858. + loff_t *ppos)
  4859. +{
  4860. + return simple_data_write(filp, ubuf, cnt, ppos, &data.max_sample);
  4861. +}
  4862. +
  4863. +
  4864. +/**
  4865. + * debug_sample_fopen - An open function for "sample" debugfs interface
  4866. + * @inode: The in-kernel inode representation of this debugfs "file"
  4867. + * @filp: The active open file structure for the debugfs "file"
  4868. + *
  4869. + * This function handles opening the "sample" file within the hardware
  4870. + * latency detector debugfs directory interface. This file is used to read
  4871. + * raw samples from the global ring_buffer and allows the user to see a
  4872. + * running latency history. Can be opened blocking or non-blocking,
  4873. + * affecting whether it behaves as a buffer read pipe, or does not.
  4874. + * Implements simple locking to prevent multiple simultaneous use.
  4875. + */
  4876. +static int debug_sample_fopen(struct inode *inode, struct file *filp)
  4877. +{
  4878. + if (!atomic_add_unless(&data.sample_open, 1, 1))
  4879. + return -EBUSY;
  4880. + else
  4881. + return 0;
  4882. +}
  4883. +
  4884. +/**
  4885. + * debug_sample_fread - A read function for "sample" debugfs interface
  4886. + * @filp: The active open file structure for the debugfs "file"
  4887. + * @ubuf: The user buffer that will contain the samples read
  4888. + * @cnt: The maximum bytes to read from the debugfs "file"
  4889. + * @ppos: The current position in the debugfs "file"
  4890. + *
  4891. + * This function handles reading from the "sample" file within the hardware
  4892. + * latency detector debugfs directory interface. This file is used to read
  4893. + * raw samples from the global ring_buffer and allows the user to see a
  4894. + * running latency history. By default this will block pending a new
  4895. + * value written into the sample buffer, unless there are already a
  4896. + * number of value(s) waiting in the buffer, or the sample file was
  4897. + * previously opened in a non-blocking mode of operation.
  4898. + */
  4899. +static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf,
  4900. + size_t cnt, loff_t *ppos)
  4901. +{
  4902. + int len = 0;
  4903. + char buf[64];
  4904. + struct sample *sample = NULL;
  4905. +
  4906. + if (!enabled)
  4907. + return 0;
  4908. +
  4909. + sample = kzalloc(sizeof(struct sample), GFP_KERNEL);
  4910. + if (!sample)
  4911. + return -ENOMEM;
  4912. +
  4913. + while (!buffer_get_sample(sample)) {
  4914. +
  4915. + DEFINE_WAIT(wait);
  4916. +
  4917. + if (filp->f_flags & O_NONBLOCK) {
  4918. + len = -EAGAIN;
  4919. + goto out;
  4920. + }
  4921. +
  4922. + prepare_to_wait(&data.wq, &wait, TASK_INTERRUPTIBLE);
  4923. + schedule();
  4924. + finish_wait(&data.wq, &wait);
  4925. +
  4926. + if (signal_pending(current)) {
  4927. + len = -EINTR;
  4928. + goto out;
  4929. + }
  4930. +
  4931. + if (!enabled) { /* enable was toggled */
  4932. + len = 0;
  4933. + goto out;
  4934. + }
  4935. + }
  4936. +
  4937. + len = snprintf(buf, sizeof(buf), "%010lu.%010lu\t%llu\t%llu\n",
  4938. + sample->timestamp.tv_sec,
  4939. + sample->timestamp.tv_nsec,
  4940. + sample->duration,
  4941. + sample->outer_duration);
  4942. +
  4943. +
  4944. + /* handling partial reads is more trouble than it's worth */
  4945. + if (len > cnt)
  4946. + goto out;
  4947. +
  4948. + if (copy_to_user(ubuf, buf, len))
  4949. + len = -EFAULT;
  4950. +
  4951. +out:
  4952. + kfree(sample);
  4953. + return len;
  4954. +}
  4955. +
  4956. +/**
  4957. + * debug_sample_release - Release function for "sample" debugfs interface
  4958. + * @inode: The in-kernel inode represenation of the debugfs "file"
  4959. + * @filp: The active open file structure for the debugfs "file"
  4960. + *
  4961. + * This function completes the close of the debugfs interface "sample" file.
  4962. + * Frees the sample_open "lock" so that other users may open the interface.
  4963. + */
  4964. +static int debug_sample_release(struct inode *inode, struct file *filp)
  4965. +{
  4966. + atomic_dec(&data.sample_open);
  4967. +
  4968. + return 0;
  4969. +}
  4970. +
  4971. +/**
  4972. + * debug_threshold_fopen - Open function for "threshold" debugfs entry
  4973. + * @inode: The in-kernel inode representation of the debugfs "file"
  4974. + * @filp: The active open file structure for the debugfs "file"
  4975. + *
  4976. + * This function provides an open implementation for the "threshold" debugfs
  4977. + * interface to the hardware latency detector.
  4978. + */
  4979. +static int debug_threshold_fopen(struct inode *inode, struct file *filp)
  4980. +{
  4981. + return 0;
  4982. +}
  4983. +
  4984. +/**
  4985. + * debug_threshold_fread - Read function for "threshold" debugfs entry
  4986. + * @filp: The active open file structure for the debugfs "file"
  4987. + * @ubuf: The userspace provided buffer to read value into
  4988. + * @cnt: The maximum number of bytes to read
  4989. + * @ppos: The current "file" position
  4990. + *
  4991. + * This function provides a read implementation for the "threshold" debugfs
  4992. + * interface to the hardware latency detector. It can be used to determine
  4993. + * the current threshold level at which a latency will be recorded in the
  4994. + * global ring buffer, typically on the order of 10us.
  4995. + */
  4996. +static ssize_t debug_threshold_fread(struct file *filp, char __user *ubuf,
  4997. + size_t cnt, loff_t *ppos)
  4998. +{
  4999. + return simple_data_read(filp, ubuf, cnt, ppos, &data.threshold);
  5000. +}
  5001. +
  5002. +/**
  5003. + * debug_threshold_fwrite - Write function for "threshold" debugfs entry
  5004. + * @filp: The active open file structure for the debugfs "file"
  5005. + * @ubuf: The user buffer that contains the value to write
  5006. + * @cnt: The maximum number of bytes to write to "file"
  5007. + * @ppos: The current position in the debugfs "file"
  5008. + *
  5009. + * This function provides a write implementation for the "threshold" debugfs
  5010. + * interface to the hardware latency detector. It can be used to configure
  5011. + * the threshold level at which any subsequently detected latencies will
  5012. + * be recorded into the global ring buffer.
  5013. + */
  5014. +static ssize_t debug_threshold_fwrite(struct file *filp,
  5015. + const char __user *ubuf,
  5016. + size_t cnt,
  5017. + loff_t *ppos)
  5018. +{
  5019. + int ret;
  5020. +
  5021. + ret = simple_data_write(filp, ubuf, cnt, ppos, &data.threshold);
  5022. +
  5023. + if (enabled)
  5024. + wake_up_process(kthread);
  5025. +
  5026. + return ret;
  5027. +}
  5028. +
  5029. +/**
  5030. + * debug_width_fopen - Open function for "width" debugfs entry
  5031. + * @inode: The in-kernel inode representation of the debugfs "file"
  5032. + * @filp: The active open file structure for the debugfs "file"
  5033. + *
  5034. + * This function provides an open implementation for the "width" debugfs
  5035. + * interface to the hardware latency detector.
  5036. + */
  5037. +static int debug_width_fopen(struct inode *inode, struct file *filp)
  5038. +{
  5039. + return 0;
  5040. +}
  5041. +
  5042. +/**
  5043. + * debug_width_fread - Read function for "width" debugfs entry
  5044. + * @filp: The active open file structure for the debugfs "file"
  5045. + * @ubuf: The userspace provided buffer to read value into
  5046. + * @cnt: The maximum number of bytes to read
  5047. + * @ppos: The current "file" position
  5048. + *
  5049. + * This function provides a read implementation for the "width" debugfs
  5050. + * interface to the hardware latency detector. It can be used to determine
  5051. + * for how many us of the total window us we will actively sample for any
  5052. + * hardware-induced latecy periods. Obviously, it is not possible to
  5053. + * sample constantly and have the system respond to a sample reader, or,
  5054. + * worse, without having the system appear to have gone out to lunch.
  5055. + */
  5056. +static ssize_t debug_width_fread(struct file *filp, char __user *ubuf,
  5057. + size_t cnt, loff_t *ppos)
  5058. +{
  5059. + return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_width);
  5060. +}
  5061. +
  5062. +/**
  5063. + * debug_width_fwrite - Write function for "width" debugfs entry
  5064. + * @filp: The active open file structure for the debugfs "file"
  5065. + * @ubuf: The user buffer that contains the value to write
  5066. + * @cnt: The maximum number of bytes to write to "file"
  5067. + * @ppos: The current position in the debugfs "file"
  5068. + *
  5069. + * This function provides a write implementation for the "width" debugfs
  5070. + * interface to the hardware latency detector. It can be used to configure
  5071. + * for how many us of the total window us we will actively sample for any
  5072. + * hardware-induced latency periods. Obviously, it is not possible to
  5073. + * sample constantly and have the system respond to a sample reader, or,
  5074. + * worse, without having the system appear to have gone out to lunch. It
  5075. + * is enforced that width is less that the total window size.
  5076. + */
  5077. +static ssize_t debug_width_fwrite(struct file *filp,
  5078. + const char __user *ubuf,
  5079. + size_t cnt,
  5080. + loff_t *ppos)
  5081. +{
  5082. + char buf[U64STR_SIZE];
  5083. + int csize = min(cnt, sizeof(buf));
  5084. + u64 val = 0;
  5085. + int err = 0;
  5086. +
  5087. + memset(buf, '\0', sizeof(buf));
  5088. + if (copy_from_user(buf, ubuf, csize))
  5089. + return -EFAULT;
  5090. +
  5091. + buf[U64STR_SIZE-1] = '\0'; /* just in case */
  5092. + err = kstrtoull(buf, 10, &val);
  5093. + if (err)
  5094. + return -EINVAL;
  5095. +
  5096. + mutex_lock(&data.lock);
  5097. + if (val < data.sample_window)
  5098. + data.sample_width = val;
  5099. + else {
  5100. + mutex_unlock(&data.lock);
  5101. + return -EINVAL;
  5102. + }
  5103. + mutex_unlock(&data.lock);
  5104. +
  5105. + if (enabled)
  5106. + wake_up_process(kthread);
  5107. +
  5108. + return csize;
  5109. +}
  5110. +
  5111. +/**
  5112. + * debug_window_fopen - Open function for "window" debugfs entry
  5113. + * @inode: The in-kernel inode representation of the debugfs "file"
  5114. + * @filp: The active open file structure for the debugfs "file"
  5115. + *
  5116. + * This function provides an open implementation for the "window" debugfs
  5117. + * interface to the hardware latency detector. The window is the total time
  5118. + * in us that will be considered one sample period. Conceptually, windows
  5119. + * occur back-to-back and contain a sample width period during which
  5120. + * actual sampling occurs.
  5121. + */
  5122. +static int debug_window_fopen(struct inode *inode, struct file *filp)
  5123. +{
  5124. + return 0;
  5125. +}
  5126. +
  5127. +/**
  5128. + * debug_window_fread - Read function for "window" debugfs entry
  5129. + * @filp: The active open file structure for the debugfs "file"
  5130. + * @ubuf: The userspace provided buffer to read value into
  5131. + * @cnt: The maximum number of bytes to read
  5132. + * @ppos: The current "file" position
  5133. + *
  5134. + * This function provides a read implementation for the "window" debugfs
  5135. + * interface to the hardware latency detector. The window is the total time
  5136. + * in us that will be considered one sample period. Conceptually, windows
  5137. + * occur back-to-back and contain a sample width period during which
  5138. + * actual sampling occurs. Can be used to read the total window size.
  5139. + */
  5140. +static ssize_t debug_window_fread(struct file *filp, char __user *ubuf,
  5141. + size_t cnt, loff_t *ppos)
  5142. +{
  5143. + return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_window);
  5144. +}
  5145. +
  5146. +/**
  5147. + * debug_window_fwrite - Write function for "window" debugfs entry
  5148. + * @filp: The active open file structure for the debugfs "file"
  5149. + * @ubuf: The user buffer that contains the value to write
  5150. + * @cnt: The maximum number of bytes to write to "file"
  5151. + * @ppos: The current position in the debugfs "file"
  5152. + *
  5153. + * This function provides a write implementation for the "window" debufds
  5154. + * interface to the hardware latency detetector. The window is the total time
  5155. + * in us that will be considered one sample period. Conceptually, windows
  5156. + * occur back-to-back and contain a sample width period during which
  5157. + * actual sampling occurs. Can be used to write a new total window size. It
  5158. + * is enfoced that any value written must be greater than the sample width
  5159. + * size, or an error results.
  5160. + */
  5161. +static ssize_t debug_window_fwrite(struct file *filp,
  5162. + const char __user *ubuf,
  5163. + size_t cnt,
  5164. + loff_t *ppos)
  5165. +{
  5166. + char buf[U64STR_SIZE];
  5167. + int csize = min(cnt, sizeof(buf));
  5168. + u64 val = 0;
  5169. + int err = 0;
  5170. +
  5171. + memset(buf, '\0', sizeof(buf));
  5172. + if (copy_from_user(buf, ubuf, csize))
  5173. + return -EFAULT;
  5174. +
  5175. + buf[U64STR_SIZE-1] = '\0'; /* just in case */
  5176. + err = kstrtoull(buf, 10, &val);
  5177. + if (err)
  5178. + return -EINVAL;
  5179. +
  5180. + mutex_lock(&data.lock);
  5181. + if (data.sample_width < val)
  5182. + data.sample_window = val;
  5183. + else {
  5184. + mutex_unlock(&data.lock);
  5185. + return -EINVAL;
  5186. + }
  5187. + mutex_unlock(&data.lock);
  5188. +
  5189. + return csize;
  5190. +}
  5191. +
  5192. +/*
  5193. + * Function pointers for the "count" debugfs file operations
  5194. + */
  5195. +static const struct file_operations count_fops = {
  5196. + .open = debug_count_fopen,
  5197. + .read = debug_count_fread,
  5198. + .write = debug_count_fwrite,
  5199. + .owner = THIS_MODULE,
  5200. +};
  5201. +
  5202. +/*
  5203. + * Function pointers for the "enable" debugfs file operations
  5204. + */
  5205. +static const struct file_operations enable_fops = {
  5206. + .open = debug_enable_fopen,
  5207. + .read = debug_enable_fread,
  5208. + .write = debug_enable_fwrite,
  5209. + .owner = THIS_MODULE,
  5210. +};
  5211. +
  5212. +/*
  5213. + * Function pointers for the "max" debugfs file operations
  5214. + */
  5215. +static const struct file_operations max_fops = {
  5216. + .open = debug_max_fopen,
  5217. + .read = debug_max_fread,
  5218. + .write = debug_max_fwrite,
  5219. + .owner = THIS_MODULE,
  5220. +};
  5221. +
  5222. +/*
  5223. + * Function pointers for the "sample" debugfs file operations
  5224. + */
  5225. +static const struct file_operations sample_fops = {
  5226. + .open = debug_sample_fopen,
  5227. + .read = debug_sample_fread,
  5228. + .release = debug_sample_release,
  5229. + .owner = THIS_MODULE,
  5230. +};
  5231. +
  5232. +/*
  5233. + * Function pointers for the "threshold" debugfs file operations
  5234. + */
  5235. +static const struct file_operations threshold_fops = {
  5236. + .open = debug_threshold_fopen,
  5237. + .read = debug_threshold_fread,
  5238. + .write = debug_threshold_fwrite,
  5239. + .owner = THIS_MODULE,
  5240. +};
  5241. +
  5242. +/*
  5243. + * Function pointers for the "width" debugfs file operations
  5244. + */
  5245. +static const struct file_operations width_fops = {
  5246. + .open = debug_width_fopen,
  5247. + .read = debug_width_fread,
  5248. + .write = debug_width_fwrite,
  5249. + .owner = THIS_MODULE,
  5250. +};
  5251. +
  5252. +/*
  5253. + * Function pointers for the "window" debugfs file operations
  5254. + */
  5255. +static const struct file_operations window_fops = {
  5256. + .open = debug_window_fopen,
  5257. + .read = debug_window_fread,
  5258. + .write = debug_window_fwrite,
  5259. + .owner = THIS_MODULE,
  5260. +};
  5261. +
  5262. +/**
  5263. + * init_debugfs - A function to initialize the debugfs interface files
  5264. + *
  5265. + * This function creates entries in debugfs for "hwlat_detector", including
  5266. + * files to read values from the detector, current samples, and the
  5267. + * maximum sample that has been captured since the hardware latency
  5268. + * dectector was started.
  5269. + */
  5270. +static int init_debugfs(void)
  5271. +{
  5272. + int ret = -ENOMEM;
  5273. +
  5274. + debug_dir = debugfs_create_dir(DRVNAME, NULL);
  5275. + if (!debug_dir)
  5276. + goto err_debug_dir;
  5277. +
  5278. + debug_sample = debugfs_create_file("sample", 0444,
  5279. + debug_dir, NULL,
  5280. + &sample_fops);
  5281. + if (!debug_sample)
  5282. + goto err_sample;
  5283. +
  5284. + debug_count = debugfs_create_file("count", 0444,
  5285. + debug_dir, NULL,
  5286. + &count_fops);
  5287. + if (!debug_count)
  5288. + goto err_count;
  5289. +
  5290. + debug_max = debugfs_create_file("max", 0444,
  5291. + debug_dir, NULL,
  5292. + &max_fops);
  5293. + if (!debug_max)
  5294. + goto err_max;
  5295. +
  5296. + debug_sample_window = debugfs_create_file("window", 0644,
  5297. + debug_dir, NULL,
  5298. + &window_fops);
  5299. + if (!debug_sample_window)
  5300. + goto err_window;
  5301. +
  5302. + debug_sample_width = debugfs_create_file("width", 0644,
  5303. + debug_dir, NULL,
  5304. + &width_fops);
  5305. + if (!debug_sample_width)
  5306. + goto err_width;
  5307. +
  5308. + debug_threshold = debugfs_create_file("threshold", 0644,
  5309. + debug_dir, NULL,
  5310. + &threshold_fops);
  5311. + if (!debug_threshold)
  5312. + goto err_threshold;
  5313. +
  5314. + debug_enable = debugfs_create_file("enable", 0644,
  5315. + debug_dir, &enabled,
  5316. + &enable_fops);
  5317. + if (!debug_enable)
  5318. + goto err_enable;
  5319. +
  5320. + else {
  5321. + ret = 0;
  5322. + goto out;
  5323. + }
  5324. +
  5325. +err_enable:
  5326. + debugfs_remove(debug_threshold);
  5327. +err_threshold:
  5328. + debugfs_remove(debug_sample_width);
  5329. +err_width:
  5330. + debugfs_remove(debug_sample_window);
  5331. +err_window:
  5332. + debugfs_remove(debug_max);
  5333. +err_max:
  5334. + debugfs_remove(debug_count);
  5335. +err_count:
  5336. + debugfs_remove(debug_sample);
  5337. +err_sample:
  5338. + debugfs_remove(debug_dir);
  5339. +err_debug_dir:
  5340. +out:
  5341. + return ret;
  5342. +}
  5343. +
  5344. +/**
  5345. + * free_debugfs - A function to cleanup the debugfs file interface
  5346. + */
  5347. +static void free_debugfs(void)
  5348. +{
  5349. + /* could also use a debugfs_remove_recursive */
  5350. + debugfs_remove(debug_enable);
  5351. + debugfs_remove(debug_threshold);
  5352. + debugfs_remove(debug_sample_width);
  5353. + debugfs_remove(debug_sample_window);
  5354. + debugfs_remove(debug_max);
  5355. + debugfs_remove(debug_count);
  5356. + debugfs_remove(debug_sample);
  5357. + debugfs_remove(debug_dir);
  5358. +}
  5359. +
  5360. +/**
  5361. + * detector_init - Standard module initialization code
  5362. + */
  5363. +static int detector_init(void)
  5364. +{
  5365. + int ret = -ENOMEM;
  5366. +
  5367. + pr_info(BANNER "version %s\n", VERSION);
  5368. +
  5369. + ret = init_stats();
  5370. + if (ret)
  5371. + goto out;
  5372. +
  5373. + ret = init_debugfs();
  5374. + if (ret)
  5375. + goto err_stats;
  5376. +
  5377. + if (enabled)
  5378. + ret = start_kthread();
  5379. +
  5380. + goto out;
  5381. +
  5382. +err_stats:
  5383. + ring_buffer_free(ring_buffer);
  5384. +out:
  5385. + return ret;
  5386. +
  5387. +}
  5388. +
  5389. +/**
  5390. + * detector_exit - Standard module cleanup code
  5391. + */
  5392. +static void detector_exit(void)
  5393. +{
  5394. + int err;
  5395. +
  5396. + if (enabled) {
  5397. + enabled = 0;
  5398. + err = stop_kthread();
  5399. + if (err)
  5400. + pr_err(BANNER "cannot stop kthread\n");
  5401. + }
  5402. +
  5403. + free_debugfs();
  5404. + ring_buffer_free(ring_buffer); /* free up the ring buffer */
  5405. +
  5406. +}
  5407. +
  5408. +module_init(detector_init);
  5409. +module_exit(detector_exit);
  5410. diff -Nur linux-4.6.6.orig/drivers/misc/Kconfig linux-4.6.6/drivers/misc/Kconfig
  5411. --- linux-4.6.6.orig/drivers/misc/Kconfig 2016-08-10 12:54:57.000000000 +0200
  5412. +++ linux-4.6.6/drivers/misc/Kconfig 2016-08-22 21:27:47.049869767 +0200
  5413. @@ -54,6 +54,7 @@
  5414. config ATMEL_TCLIB
  5415. bool "Atmel AT32/AT91 Timer/Counter Library"
  5416. depends on (AVR32 || ARCH_AT91)
  5417. + default y if PREEMPT_RT_FULL
  5418. help
  5419. Select this if you want a library to allocate the Timer/Counter
  5420. blocks found on many Atmel processors. This facilitates using
  5421. @@ -69,8 +70,7 @@
  5422. are combined to make a single 32-bit timer.
  5423. When GENERIC_CLOCKEVENTS is defined, the third timer channel
  5424. - may be used as a clock event device supporting oneshot mode
  5425. - (delays of up to two seconds) based on the 32 KiHz clock.
  5426. + may be used as a clock event device supporting oneshot mode.
  5427. config ATMEL_TCB_CLKSRC_BLOCK
  5428. int
  5429. @@ -84,6 +84,15 @@
  5430. TC can be used for other purposes, such as PWM generation and
  5431. interval timing.
  5432. +config ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK
  5433. + bool "TC Block use 32 KiHz clock"
  5434. + depends on ATMEL_TCB_CLKSRC
  5435. + default y if !PREEMPT_RT_FULL
  5436. + help
  5437. + Select this to use 32 KiHz base clock rate as TC block clock
  5438. + source for clock events.
  5439. +
  5440. +
  5441. config DUMMY_IRQ
  5442. tristate "Dummy IRQ handler"
  5443. default n
  5444. @@ -114,6 +123,35 @@
  5445. for information on the specific driver level and support statement
  5446. for your IBM server.
  5447. +config HWLAT_DETECTOR
  5448. + tristate "Testing module to detect hardware-induced latencies"
  5449. + depends on DEBUG_FS
  5450. + depends on RING_BUFFER
  5451. + default m
  5452. + ---help---
  5453. + A simple hardware latency detector. Use this module to detect
  5454. + large latencies introduced by the behavior of the underlying
  5455. + system firmware external to Linux. We do this using periodic
  5456. + use of stop_machine to grab all available CPUs and measure
  5457. + for unexplainable gaps in the CPU timestamp counter(s). By
  5458. + default, the module is not enabled until the "enable" file
  5459. + within the "hwlat_detector" debugfs directory is toggled.
  5460. +
  5461. + This module is often used to detect SMI (System Management
  5462. + Interrupts) on x86 systems, though is not x86 specific. To
  5463. + this end, we default to using a sample window of 1 second,
  5464. + during which we will sample for 0.5 seconds. If an SMI or
  5465. + similar event occurs during that time, it is recorded
  5466. + into an 8K samples global ring buffer until retreived.
  5467. +
  5468. + WARNING: This software should never be enabled (it can be built
  5469. + but should not be turned on after it is loaded) in a production
  5470. + environment where high latencies are a concern since the
  5471. + sampling mechanism actually introduces latencies for
  5472. + regular tasks while the CPU(s) are being held.
  5473. +
  5474. + If unsure, say N
  5475. +
  5476. config PHANTOM
  5477. tristate "Sensable PHANToM (PCI)"
  5478. depends on PCI
  5479. diff -Nur linux-4.6.6.orig/drivers/misc/Makefile linux-4.6.6/drivers/misc/Makefile
  5480. --- linux-4.6.6.orig/drivers/misc/Makefile 2016-08-10 12:54:57.000000000 +0200
  5481. +++ linux-4.6.6/drivers/misc/Makefile 2016-08-22 21:27:47.049869767 +0200
  5482. @@ -39,6 +39,7 @@
  5483. obj-$(CONFIG_HMC6352) += hmc6352.o
  5484. obj-y += eeprom/
  5485. obj-y += cb710/
  5486. +obj-$(CONFIG_HWLAT_DETECTOR) += hwlat_detector.o
  5487. obj-$(CONFIG_SPEAR13XX_PCIE_GADGET) += spear13xx_pcie_gadget.o
  5488. obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o
  5489. obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o
  5490. diff -Nur linux-4.6.6.orig/drivers/mmc/host/jz4740_mmc.c linux-4.6.6/drivers/mmc/host/jz4740_mmc.c
  5491. --- linux-4.6.6.orig/drivers/mmc/host/jz4740_mmc.c 2016-08-10 12:54:57.000000000 +0200
  5492. +++ linux-4.6.6/drivers/mmc/host/jz4740_mmc.c 2016-08-22 21:27:47.061870541 +0200
  5493. @@ -1068,8 +1068,6 @@
  5494. jz4740_mmc_clock_disable(host);
  5495. setup_timer(&host->timeout_timer, jz4740_mmc_timeout,
  5496. (unsigned long)host);
  5497. - /* It is not important when it times out, it just needs to timeout. */
  5498. - set_timer_slack(&host->timeout_timer, HZ);
  5499. host->use_dma = true;
  5500. if (host->use_dma && jz4740_mmc_acquire_dma_channels(host) != 0)
  5501. diff -Nur linux-4.6.6.orig/drivers/mmc/host/mmci.c linux-4.6.6/drivers/mmc/host/mmci.c
  5502. --- linux-4.6.6.orig/drivers/mmc/host/mmci.c 2016-08-10 12:54:57.000000000 +0200
  5503. +++ linux-4.6.6/drivers/mmc/host/mmci.c 2016-08-22 21:27:47.069871054 +0200
  5504. @@ -1155,15 +1155,12 @@
  5505. struct sg_mapping_iter *sg_miter = &host->sg_miter;
  5506. struct variant_data *variant = host->variant;
  5507. void __iomem *base = host->base;
  5508. - unsigned long flags;
  5509. u32 status;
  5510. status = readl(base + MMCISTATUS);
  5511. dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status);
  5512. - local_irq_save(flags);
  5513. -
  5514. do {
  5515. unsigned int remain, len;
  5516. char *buffer;
  5517. @@ -1203,8 +1200,6 @@
  5518. sg_miter_stop(sg_miter);
  5519. - local_irq_restore(flags);
  5520. -
  5521. /*
  5522. * If we have less than the fifo 'half-full' threshold to transfer,
  5523. * trigger a PIO interrupt as soon as any data is available.
  5524. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/3com/3c59x.c linux-4.6.6/drivers/net/ethernet/3com/3c59x.c
  5525. --- linux-4.6.6.orig/drivers/net/ethernet/3com/3c59x.c 2016-08-10 12:54:57.000000000 +0200
  5526. +++ linux-4.6.6/drivers/net/ethernet/3com/3c59x.c 2016-08-22 21:27:47.081871820 +0200
  5527. @@ -842,9 +842,9 @@
  5528. {
  5529. struct vortex_private *vp = netdev_priv(dev);
  5530. unsigned long flags;
  5531. - local_irq_save(flags);
  5532. + local_irq_save_nort(flags);
  5533. (vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev);
  5534. - local_irq_restore(flags);
  5535. + local_irq_restore_nort(flags);
  5536. }
  5537. #endif
  5538. @@ -1910,12 +1910,12 @@
  5539. * Block interrupts because vortex_interrupt does a bare spin_lock()
  5540. */
  5541. unsigned long flags;
  5542. - local_irq_save(flags);
  5543. + local_irq_save_nort(flags);
  5544. if (vp->full_bus_master_tx)
  5545. boomerang_interrupt(dev->irq, dev);
  5546. else
  5547. vortex_interrupt(dev->irq, dev);
  5548. - local_irq_restore(flags);
  5549. + local_irq_restore_nort(flags);
  5550. }
  5551. }
  5552. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.6.6/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
  5553. --- linux-4.6.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2016-08-10 12:54:57.000000000 +0200
  5554. +++ linux-4.6.6/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2016-08-22 21:27:47.093872593 +0200
  5555. @@ -2217,11 +2217,7 @@
  5556. }
  5557. tpd_req = atl1c_cal_tpd_req(skb);
  5558. - if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) {
  5559. - if (netif_msg_pktdata(adapter))
  5560. - dev_info(&adapter->pdev->dev, "tx locked\n");
  5561. - return NETDEV_TX_LOCKED;
  5562. - }
  5563. + spin_lock_irqsave(&adapter->tx_lock, flags);
  5564. if (atl1c_tpd_avail(adapter, type) < tpd_req) {
  5565. /* no enough descriptor, just stop queue */
  5566. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.6.6/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
  5567. --- linux-4.6.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2016-08-10 12:54:57.000000000 +0200
  5568. +++ linux-4.6.6/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2016-08-22 21:27:47.101873106 +0200
  5569. @@ -1880,8 +1880,7 @@
  5570. return NETDEV_TX_OK;
  5571. }
  5572. tpd_req = atl1e_cal_tdp_req(skb);
  5573. - if (!spin_trylock_irqsave(&adapter->tx_lock, flags))
  5574. - return NETDEV_TX_LOCKED;
  5575. + spin_lock_irqsave(&adapter->tx_lock, flags);
  5576. if (atl1e_tpd_avail(adapter) < tpd_req) {
  5577. /* no enough descriptor, just stop queue */
  5578. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.6.6/drivers/net/ethernet/chelsio/cxgb/sge.c
  5579. --- linux-4.6.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2016-08-10 12:54:57.000000000 +0200
  5580. +++ linux-4.6.6/drivers/net/ethernet/chelsio/cxgb/sge.c 2016-08-22 21:27:47.113873872 +0200
  5581. @@ -1664,8 +1664,7 @@
  5582. struct cmdQ *q = &sge->cmdQ[qid];
  5583. unsigned int credits, pidx, genbit, count, use_sched_skb = 0;
  5584. - if (!spin_trylock(&q->lock))
  5585. - return NETDEV_TX_LOCKED;
  5586. + spin_lock(&q->lock);
  5587. reclaim_completed_tx(sge, q);
  5588. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/neterion/s2io.c linux-4.6.6/drivers/net/ethernet/neterion/s2io.c
  5589. --- linux-4.6.6.orig/drivers/net/ethernet/neterion/s2io.c 2016-08-10 12:54:57.000000000 +0200
  5590. +++ linux-4.6.6/drivers/net/ethernet/neterion/s2io.c 2016-08-22 21:27:47.141875672 +0200
  5591. @@ -4084,12 +4084,7 @@
  5592. [skb->priority & (MAX_TX_FIFOS - 1)];
  5593. fifo = &mac_control->fifos[queue];
  5594. - if (do_spin_lock)
  5595. - spin_lock_irqsave(&fifo->tx_lock, flags);
  5596. - else {
  5597. - if (unlikely(!spin_trylock_irqsave(&fifo->tx_lock, flags)))
  5598. - return NETDEV_TX_LOCKED;
  5599. - }
  5600. + spin_lock_irqsave(&fifo->tx_lock, flags);
  5601. if (sp->config.multiq) {
  5602. if (__netif_subqueue_stopped(dev, fifo->fifo_no)) {
  5603. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.6.6/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
  5604. --- linux-4.6.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2016-08-10 12:54:57.000000000 +0200
  5605. +++ linux-4.6.6/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2016-08-22 21:27:47.149876185 +0200
  5606. @@ -2137,10 +2137,8 @@
  5607. struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring;
  5608. unsigned long flags;
  5609. - if (!spin_trylock_irqsave(&tx_ring->tx_lock, flags)) {
  5610. - /* Collision - tell upper layer to requeue */
  5611. - return NETDEV_TX_LOCKED;
  5612. - }
  5613. + spin_lock_irqsave(&tx_ring->tx_lock, flags);
  5614. +
  5615. if (unlikely(!PCH_GBE_DESC_UNUSED(tx_ring))) {
  5616. netif_stop_queue(netdev);
  5617. spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
  5618. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/realtek/8139too.c linux-4.6.6/drivers/net/ethernet/realtek/8139too.c
  5619. --- linux-4.6.6.orig/drivers/net/ethernet/realtek/8139too.c 2016-08-10 12:54:57.000000000 +0200
  5620. +++ linux-4.6.6/drivers/net/ethernet/realtek/8139too.c 2016-08-22 21:27:47.157876698 +0200
  5621. @@ -2229,7 +2229,7 @@
  5622. struct rtl8139_private *tp = netdev_priv(dev);
  5623. const int irq = tp->pci_dev->irq;
  5624. - disable_irq(irq);
  5625. + disable_irq_nosync(irq);
  5626. rtl8139_interrupt(irq, dev);
  5627. enable_irq(irq);
  5628. }
  5629. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.6.6/drivers/net/ethernet/tehuti/tehuti.c
  5630. --- linux-4.6.6.orig/drivers/net/ethernet/tehuti/tehuti.c 2016-08-10 12:54:57.000000000 +0200
  5631. +++ linux-4.6.6/drivers/net/ethernet/tehuti/tehuti.c 2016-08-22 21:27:47.165877211 +0200
  5632. @@ -1629,13 +1629,8 @@
  5633. unsigned long flags;
  5634. ENTER;
  5635. - local_irq_save(flags);
  5636. - if (!spin_trylock(&priv->tx_lock)) {
  5637. - local_irq_restore(flags);
  5638. - DBG("%s[%s]: TX locked, returning NETDEV_TX_LOCKED\n",
  5639. - BDX_DRV_NAME, ndev->name);
  5640. - return NETDEV_TX_LOCKED;
  5641. - }
  5642. +
  5643. + spin_lock_irqsave(&priv->tx_lock, flags);
  5644. /* build tx descriptor */
  5645. BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */
  5646. diff -Nur linux-4.6.6.orig/drivers/net/ethernet/tile/tilepro.c linux-4.6.6/drivers/net/ethernet/tile/tilepro.c
  5647. --- linux-4.6.6.orig/drivers/net/ethernet/tile/tilepro.c 2016-08-10 12:54:57.000000000 +0200
  5648. +++ linux-4.6.6/drivers/net/ethernet/tile/tilepro.c 2016-08-22 21:27:47.173877724 +0200
  5649. @@ -588,7 +588,7 @@
  5650. static void tile_net_schedule_egress_timer(struct tile_net_cpu *info)
  5651. {
  5652. if (!info->egress_timer_scheduled) {
  5653. - mod_timer_pinned(&info->egress_timer, jiffies + 1);
  5654. + mod_timer(&info->egress_timer, jiffies + 1);
  5655. info->egress_timer_scheduled = true;
  5656. }
  5657. }
  5658. @@ -1004,7 +1004,7 @@
  5659. BUG();
  5660. /* Initialize the egress timer. */
  5661. - init_timer(&info->egress_timer);
  5662. + init_timer_pinned(&info->egress_timer);
  5663. info->egress_timer.data = (long)info;
  5664. info->egress_timer.function = tile_net_handle_egress_timer;
  5665. diff -Nur linux-4.6.6.orig/drivers/net/rionet.c linux-4.6.6/drivers/net/rionet.c
  5666. --- linux-4.6.6.orig/drivers/net/rionet.c 2016-08-10 12:54:57.000000000 +0200
  5667. +++ linux-4.6.6/drivers/net/rionet.c 2016-08-22 21:27:47.181878237 +0200
  5668. @@ -179,11 +179,7 @@
  5669. unsigned long flags;
  5670. int add_num = 1;
  5671. - local_irq_save(flags);
  5672. - if (!spin_trylock(&rnet->tx_lock)) {
  5673. - local_irq_restore(flags);
  5674. - return NETDEV_TX_LOCKED;
  5675. - }
  5676. + spin_lock_irqsave(&rnet->tx_lock, flags);
  5677. if (is_multicast_ether_addr(eth->h_dest))
  5678. add_num = nets[rnet->mport->id].nact;
  5679. diff -Nur linux-4.6.6.orig/drivers/net/wireless/intersil/orinoco/orinoco_usb.c linux-4.6.6/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
  5680. --- linux-4.6.6.orig/drivers/net/wireless/intersil/orinoco/orinoco_usb.c 2016-08-10 12:54:57.000000000 +0200
  5681. +++ linux-4.6.6/drivers/net/wireless/intersil/orinoco/orinoco_usb.c 2016-08-22 21:27:47.193879004 +0200
  5682. @@ -697,7 +697,7 @@
  5683. while (!ctx->done.done && msecs--)
  5684. udelay(1000);
  5685. } else {
  5686. - wait_event_interruptible(ctx->done.wait,
  5687. + swait_event_interruptible(ctx->done.wait,
  5688. ctx->done.done);
  5689. }
  5690. break;
  5691. diff -Nur linux-4.6.6.orig/drivers/pci/access.c linux-4.6.6/drivers/pci/access.c
  5692. --- linux-4.6.6.orig/drivers/pci/access.c 2016-08-10 12:54:57.000000000 +0200
  5693. +++ linux-4.6.6/drivers/pci/access.c 2016-08-22 21:27:47.201879517 +0200
  5694. @@ -672,7 +672,7 @@
  5695. WARN_ON(!dev->block_cfg_access);
  5696. dev->block_cfg_access = 0;
  5697. - wake_up_all(&pci_cfg_wait);
  5698. + wake_up_all_locked(&pci_cfg_wait);
  5699. raw_spin_unlock_irqrestore(&pci_lock, flags);
  5700. }
  5701. EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
  5702. diff -Nur linux-4.6.6.orig/drivers/power/bq27xxx_battery.c linux-4.6.6/drivers/power/bq27xxx_battery.c
  5703. --- linux-4.6.6.orig/drivers/power/bq27xxx_battery.c 2016-08-10 12:54:57.000000000 +0200
  5704. +++ linux-4.6.6/drivers/power/bq27xxx_battery.c 2016-08-22 21:27:47.213880286 +0200
  5705. @@ -735,11 +735,8 @@
  5706. bq27xxx_battery_update(di);
  5707. - if (poll_interval > 0) {
  5708. - /* The timer does not have to be accurate. */
  5709. - set_timer_slack(&di->work.timer, poll_interval * HZ / 4);
  5710. + if (poll_interval > 0)
  5711. schedule_delayed_work(&di->work, poll_interval * HZ);
  5712. - }
  5713. }
  5714. /*
  5715. diff -Nur linux-4.6.6.orig/drivers/scsi/fcoe/fcoe.c linux-4.6.6/drivers/scsi/fcoe/fcoe.c
  5716. --- linux-4.6.6.orig/drivers/scsi/fcoe/fcoe.c 2016-08-10 12:54:57.000000000 +0200
  5717. +++ linux-4.6.6/drivers/scsi/fcoe/fcoe.c 2016-08-22 21:27:47.221880796 +0200
  5718. @@ -1286,7 +1286,7 @@
  5719. struct sk_buff *skb;
  5720. #ifdef CONFIG_SMP
  5721. struct fcoe_percpu_s *p0;
  5722. - unsigned targ_cpu = get_cpu();
  5723. + unsigned targ_cpu = get_cpu_light();
  5724. #endif /* CONFIG_SMP */
  5725. FCOE_DBG("Destroying receive thread for CPU %d\n", cpu);
  5726. @@ -1342,7 +1342,7 @@
  5727. kfree_skb(skb);
  5728. spin_unlock_bh(&p->fcoe_rx_list.lock);
  5729. }
  5730. - put_cpu();
  5731. + put_cpu_light();
  5732. #else
  5733. /*
  5734. * This a non-SMP scenario where the singular Rx thread is
  5735. @@ -1566,11 +1566,11 @@
  5736. static int fcoe_alloc_paged_crc_eof(struct sk_buff *skb, int tlen)
  5737. {
  5738. struct fcoe_percpu_s *fps;
  5739. - int rc;
  5740. + int rc, cpu = get_cpu_light();
  5741. - fps = &get_cpu_var(fcoe_percpu);
  5742. + fps = &per_cpu(fcoe_percpu, cpu);
  5743. rc = fcoe_get_paged_crc_eof(skb, tlen, fps);
  5744. - put_cpu_var(fcoe_percpu);
  5745. + put_cpu_light();
  5746. return rc;
  5747. }
  5748. @@ -1766,11 +1766,11 @@
  5749. return 0;
  5750. }
  5751. - stats = per_cpu_ptr(lport->stats, get_cpu());
  5752. + stats = per_cpu_ptr(lport->stats, get_cpu_light());
  5753. stats->InvalidCRCCount++;
  5754. if (stats->InvalidCRCCount < 5)
  5755. printk(KERN_WARNING "fcoe: dropping frame with CRC error\n");
  5756. - put_cpu();
  5757. + put_cpu_light();
  5758. return -EINVAL;
  5759. }
  5760. @@ -1814,7 +1814,7 @@
  5761. */
  5762. hp = (struct fcoe_hdr *) skb_network_header(skb);
  5763. - stats = per_cpu_ptr(lport->stats, get_cpu());
  5764. + stats = per_cpu_ptr(lport->stats, get_cpu_light());
  5765. if (unlikely(FC_FCOE_DECAPS_VER(hp) != FC_FCOE_VER)) {
  5766. if (stats->ErrorFrames < 5)
  5767. printk(KERN_WARNING "fcoe: FCoE version "
  5768. @@ -1846,13 +1846,13 @@
  5769. goto drop;
  5770. if (!fcoe_filter_frames(lport, fp)) {
  5771. - put_cpu();
  5772. + put_cpu_light();
  5773. fc_exch_recv(lport, fp);
  5774. return;
  5775. }
  5776. drop:
  5777. stats->ErrorFrames++;
  5778. - put_cpu();
  5779. + put_cpu_light();
  5780. kfree_skb(skb);
  5781. }
  5782. diff -Nur linux-4.6.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.6.6/drivers/scsi/fcoe/fcoe_ctlr.c
  5783. --- linux-4.6.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2016-08-10 12:54:57.000000000 +0200
  5784. +++ linux-4.6.6/drivers/scsi/fcoe/fcoe_ctlr.c 2016-08-22 21:27:47.233881568 +0200
  5785. @@ -831,7 +831,7 @@
  5786. INIT_LIST_HEAD(&del_list);
  5787. - stats = per_cpu_ptr(fip->lp->stats, get_cpu());
  5788. + stats = per_cpu_ptr(fip->lp->stats, get_cpu_light());
  5789. list_for_each_entry_safe(fcf, next, &fip->fcfs, list) {
  5790. deadline = fcf->time + fcf->fka_period + fcf->fka_period / 2;
  5791. @@ -867,7 +867,7 @@
  5792. sel_time = fcf->time;
  5793. }
  5794. }
  5795. - put_cpu();
  5796. + put_cpu_light();
  5797. list_for_each_entry_safe(fcf, next, &del_list, list) {
  5798. /* Removes fcf from current list */
  5799. diff -Nur linux-4.6.6.orig/drivers/scsi/libfc/fc_exch.c linux-4.6.6/drivers/scsi/libfc/fc_exch.c
  5800. --- linux-4.6.6.orig/drivers/scsi/libfc/fc_exch.c 2016-08-10 12:54:57.000000000 +0200
  5801. +++ linux-4.6.6/drivers/scsi/libfc/fc_exch.c 2016-08-22 21:27:47.245882341 +0200
  5802. @@ -814,10 +814,10 @@
  5803. }
  5804. memset(ep, 0, sizeof(*ep));
  5805. - cpu = get_cpu();
  5806. + cpu = get_cpu_light();
  5807. pool = per_cpu_ptr(mp->pool, cpu);
  5808. spin_lock_bh(&pool->lock);
  5809. - put_cpu();
  5810. + put_cpu_light();
  5811. /* peek cache of free slot */
  5812. if (pool->left != FC_XID_UNKNOWN) {
  5813. diff -Nur linux-4.6.6.orig/drivers/scsi/libsas/sas_ata.c linux-4.6.6/drivers/scsi/libsas/sas_ata.c
  5814. --- linux-4.6.6.orig/drivers/scsi/libsas/sas_ata.c 2016-08-10 12:54:57.000000000 +0200
  5815. +++ linux-4.6.6/drivers/scsi/libsas/sas_ata.c 2016-08-22 21:27:47.253882854 +0200
  5816. @@ -190,7 +190,7 @@
  5817. /* TODO: audit callers to ensure they are ready for qc_issue to
  5818. * unconditionally re-enable interrupts
  5819. */
  5820. - local_irq_save(flags);
  5821. + local_irq_save_nort(flags);
  5822. spin_unlock(ap->lock);
  5823. /* If the device fell off, no sense in issuing commands */
  5824. @@ -255,7 +255,7 @@
  5825. out:
  5826. spin_lock(ap->lock);
  5827. - local_irq_restore(flags);
  5828. + local_irq_restore_nort(flags);
  5829. return ret;
  5830. }
  5831. diff -Nur linux-4.6.6.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.6.6/drivers/scsi/qla2xxx/qla_inline.h
  5832. --- linux-4.6.6.orig/drivers/scsi/qla2xxx/qla_inline.h 2016-08-10 12:54:57.000000000 +0200
  5833. +++ linux-4.6.6/drivers/scsi/qla2xxx/qla_inline.h 2016-08-22 21:27:47.261883367 +0200
  5834. @@ -59,12 +59,12 @@
  5835. {
  5836. unsigned long flags;
  5837. struct qla_hw_data *ha = rsp->hw;
  5838. - local_irq_save(flags);
  5839. + local_irq_save_nort(flags);
  5840. if (IS_P3P_TYPE(ha))
  5841. qla82xx_poll(0, rsp);
  5842. else
  5843. ha->isp_ops->intr_handler(0, rsp);
  5844. - local_irq_restore(flags);
  5845. + local_irq_restore_nort(flags);
  5846. }
  5847. static inline uint8_t *
  5848. diff -Nur linux-4.6.6.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.6.6/drivers/thermal/x86_pkg_temp_thermal.c
  5849. --- linux-4.6.6.orig/drivers/thermal/x86_pkg_temp_thermal.c 2016-08-10 12:54:57.000000000 +0200
  5850. +++ linux-4.6.6/drivers/thermal/x86_pkg_temp_thermal.c 2016-08-22 21:27:47.273884133 +0200
  5851. @@ -29,6 +29,7 @@
  5852. #include <linux/pm.h>
  5853. #include <linux/thermal.h>
  5854. #include <linux/debugfs.h>
  5855. +#include <linux/swork.h>
  5856. #include <asm/cpu_device_id.h>
  5857. #include <asm/mce.h>
  5858. @@ -352,7 +353,7 @@
  5859. }
  5860. }
  5861. -static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
  5862. +static void platform_thermal_notify_work(struct swork_event *event)
  5863. {
  5864. unsigned long flags;
  5865. int cpu = smp_processor_id();
  5866. @@ -369,7 +370,7 @@
  5867. pkg_work_scheduled[phy_id]) {
  5868. disable_pkg_thres_interrupt();
  5869. spin_unlock_irqrestore(&pkg_work_lock, flags);
  5870. - return -EINVAL;
  5871. + return;
  5872. }
  5873. pkg_work_scheduled[phy_id] = 1;
  5874. spin_unlock_irqrestore(&pkg_work_lock, flags);
  5875. @@ -378,9 +379,48 @@
  5876. schedule_delayed_work_on(cpu,
  5877. &per_cpu(pkg_temp_thermal_threshold_work, cpu),
  5878. msecs_to_jiffies(notify_delay_ms));
  5879. +}
  5880. +
  5881. +#ifdef CONFIG_PREEMPT_RT_FULL
  5882. +static struct swork_event notify_work;
  5883. +
  5884. +static int thermal_notify_work_init(void)
  5885. +{
  5886. + int err;
  5887. +
  5888. + err = swork_get();
  5889. + if (err)
  5890. + return err;
  5891. +
  5892. + INIT_SWORK(&notify_work, platform_thermal_notify_work);
  5893. return 0;
  5894. }
  5895. +static void thermal_notify_work_cleanup(void)
  5896. +{
  5897. + swork_put();
  5898. +}
  5899. +
  5900. +static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
  5901. +{
  5902. + swork_queue(&notify_work);
  5903. + return 0;
  5904. +}
  5905. +
  5906. +#else /* !CONFIG_PREEMPT_RT_FULL */
  5907. +
  5908. +static int thermal_notify_work_init(void) { return 0; }
  5909. +
  5910. +static void thermal_notify_work_cleanup(void) { }
  5911. +
  5912. +static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
  5913. +{
  5914. + platform_thermal_notify_work(NULL);
  5915. +
  5916. + return 0;
  5917. +}
  5918. +#endif /* CONFIG_PREEMPT_RT_FULL */
  5919. +
  5920. static int find_siblings_cpu(int cpu)
  5921. {
  5922. int i;
  5923. @@ -584,6 +624,9 @@
  5924. if (!x86_match_cpu(pkg_temp_thermal_ids))
  5925. return -ENODEV;
  5926. + if (!thermal_notify_work_init())
  5927. + return -ENODEV;
  5928. +
  5929. spin_lock_init(&pkg_work_lock);
  5930. platform_thermal_package_notify =
  5931. pkg_temp_thermal_platform_thermal_notify;
  5932. @@ -608,7 +651,7 @@
  5933. kfree(pkg_work_scheduled);
  5934. platform_thermal_package_notify = NULL;
  5935. platform_thermal_package_rate_control = NULL;
  5936. -
  5937. + thermal_notify_work_cleanup();
  5938. return -ENODEV;
  5939. }
  5940. @@ -633,6 +676,7 @@
  5941. mutex_unlock(&phy_dev_list_mutex);
  5942. platform_thermal_package_notify = NULL;
  5943. platform_thermal_package_rate_control = NULL;
  5944. + thermal_notify_work_cleanup();
  5945. for_each_online_cpu(i)
  5946. cancel_delayed_work_sync(
  5947. &per_cpu(pkg_temp_thermal_threshold_work, i));
  5948. diff -Nur linux-4.6.6.orig/drivers/tty/metag_da.c linux-4.6.6/drivers/tty/metag_da.c
  5949. --- linux-4.6.6.orig/drivers/tty/metag_da.c 2016-08-10 12:54:57.000000000 +0200
  5950. +++ linux-4.6.6/drivers/tty/metag_da.c 2016-08-22 21:27:47.273884133 +0200
  5951. @@ -323,12 +323,12 @@
  5952. if (channel >= 0)
  5953. fetch_data(channel);
  5954. - mod_timer_pinned(&poll_timer, jiffies + DA_TTY_POLL);
  5955. + mod_timer(&poll_timer, jiffies + DA_TTY_POLL);
  5956. }
  5957. static void add_poll_timer(struct timer_list *poll_timer)
  5958. {
  5959. - setup_timer(poll_timer, dashtty_timer, 0);
  5960. + setup_pinned_timer(poll_timer, dashtty_timer, 0);
  5961. poll_timer->expires = jiffies + DA_TTY_POLL;
  5962. /*
  5963. diff -Nur linux-4.6.6.orig/drivers/tty/mips_ejtag_fdc.c linux-4.6.6/drivers/tty/mips_ejtag_fdc.c
  5964. --- linux-4.6.6.orig/drivers/tty/mips_ejtag_fdc.c 2016-08-10 12:54:57.000000000 +0200
  5965. +++ linux-4.6.6/drivers/tty/mips_ejtag_fdc.c 2016-08-22 21:27:47.273884133 +0200
  5966. @@ -689,7 +689,7 @@
  5967. mips_ejtag_fdc_handle(priv);
  5968. if (!priv->removing)
  5969. - mod_timer_pinned(&priv->poll_timer, jiffies + FDC_TTY_POLL);
  5970. + mod_timer(&priv->poll_timer, jiffies + FDC_TTY_POLL);
  5971. }
  5972. /* TTY Port operations */
  5973. @@ -1002,7 +1002,7 @@
  5974. raw_spin_unlock_irq(&priv->lock);
  5975. } else {
  5976. /* If we didn't get an usable IRQ, poll instead */
  5977. - setup_timer(&priv->poll_timer, mips_ejtag_fdc_tty_timer,
  5978. + setup_pinned_timer(&priv->poll_timer, mips_ejtag_fdc_tty_timer,
  5979. (unsigned long)priv);
  5980. priv->poll_timer.expires = jiffies + FDC_TTY_POLL;
  5981. /*
  5982. diff -Nur linux-4.6.6.orig/drivers/tty/serial/8250/8250_core.c linux-4.6.6/drivers/tty/serial/8250/8250_core.c
  5983. --- linux-4.6.6.orig/drivers/tty/serial/8250/8250_core.c 2016-08-10 12:54:57.000000000 +0200
  5984. +++ linux-4.6.6/drivers/tty/serial/8250/8250_core.c 2016-08-22 21:27:47.273884133 +0200
  5985. @@ -58,7 +58,16 @@
  5986. static unsigned int skip_txen_test; /* force skip of txen test at init time */
  5987. -#define PASS_LIMIT 512
  5988. +/*
  5989. + * On -rt we can have a more delays, and legitimately
  5990. + * so - so don't drop work spuriously and spam the
  5991. + * syslog:
  5992. + */
  5993. +#ifdef CONFIG_PREEMPT_RT_FULL
  5994. +# define PASS_LIMIT 1000000
  5995. +#else
  5996. +# define PASS_LIMIT 512
  5997. +#endif
  5998. #include <asm/serial.h>
  5999. /*
  6000. diff -Nur linux-4.6.6.orig/drivers/tty/serial/8250/8250_port.c linux-4.6.6/drivers/tty/serial/8250/8250_port.c
  6001. --- linux-4.6.6.orig/drivers/tty/serial/8250/8250_port.c 2016-08-10 12:54:57.000000000 +0200
  6002. +++ linux-4.6.6/drivers/tty/serial/8250/8250_port.c 2016-08-22 21:27:47.277884393 +0200
  6003. @@ -35,6 +35,7 @@
  6004. #include <linux/nmi.h>
  6005. #include <linux/mutex.h>
  6006. #include <linux/slab.h>
  6007. +#include <linux/kdb.h>
  6008. #include <linux/uaccess.h>
  6009. #include <linux/pm_runtime.h>
  6010. #include <linux/timer.h>
  6011. @@ -3092,9 +3093,9 @@
  6012. serial8250_rpm_get(up);
  6013. - if (port->sysrq)
  6014. + if (port->sysrq || oops_in_progress)
  6015. locked = 0;
  6016. - else if (oops_in_progress)
  6017. + else if (in_kdb_printk())
  6018. locked = spin_trylock_irqsave(&port->lock, flags);
  6019. else
  6020. spin_lock_irqsave(&port->lock, flags);
  6021. diff -Nur linux-4.6.6.orig/drivers/tty/serial/amba-pl011.c linux-4.6.6/drivers/tty/serial/amba-pl011.c
  6022. --- linux-4.6.6.orig/drivers/tty/serial/amba-pl011.c 2016-08-10 12:54:57.000000000 +0200
  6023. +++ linux-4.6.6/drivers/tty/serial/amba-pl011.c 2016-08-22 21:27:47.277884393 +0200
  6024. @@ -2166,13 +2166,19 @@
  6025. clk_enable(uap->clk);
  6026. - local_irq_save(flags);
  6027. + /*
  6028. + * local_irq_save(flags);
  6029. + *
  6030. + * This local_irq_save() is nonsense. If we come in via sysrq
  6031. + * handling then interrupts are already disabled. Aside of
  6032. + * that the port.sysrq check is racy on SMP regardless.
  6033. + */
  6034. if (uap->port.sysrq)
  6035. locked = 0;
  6036. else if (oops_in_progress)
  6037. - locked = spin_trylock(&uap->port.lock);
  6038. + locked = spin_trylock_irqsave(&uap->port.lock, flags);
  6039. else
  6040. - spin_lock(&uap->port.lock);
  6041. + spin_lock_irqsave(&uap->port.lock, flags);
  6042. /*
  6043. * First save the CR then disable the interrupts
  6044. @@ -2196,8 +2202,7 @@
  6045. pl011_write(old_cr, uap, REG_CR);
  6046. if (locked)
  6047. - spin_unlock(&uap->port.lock);
  6048. - local_irq_restore(flags);
  6049. + spin_unlock_irqrestore(&uap->port.lock, flags);
  6050. clk_disable(uap->clk);
  6051. }
  6052. diff -Nur linux-4.6.6.orig/drivers/tty/serial/omap-serial.c linux-4.6.6/drivers/tty/serial/omap-serial.c
  6053. --- linux-4.6.6.orig/drivers/tty/serial/omap-serial.c 2016-08-10 12:54:57.000000000 +0200
  6054. +++ linux-4.6.6/drivers/tty/serial/omap-serial.c 2016-08-22 21:27:47.281884646 +0200
  6055. @@ -1257,13 +1257,10 @@
  6056. pm_runtime_get_sync(up->dev);
  6057. - local_irq_save(flags);
  6058. - if (up->port.sysrq)
  6059. - locked = 0;
  6060. - else if (oops_in_progress)
  6061. - locked = spin_trylock(&up->port.lock);
  6062. + if (up->port.sysrq || oops_in_progress)
  6063. + locked = spin_trylock_irqsave(&up->port.lock, flags);
  6064. else
  6065. - spin_lock(&up->port.lock);
  6066. + spin_lock_irqsave(&up->port.lock, flags);
  6067. /*
  6068. * First save the IER then disable the interrupts
  6069. @@ -1292,8 +1289,7 @@
  6070. pm_runtime_mark_last_busy(up->dev);
  6071. pm_runtime_put_autosuspend(up->dev);
  6072. if (locked)
  6073. - spin_unlock(&up->port.lock);
  6074. - local_irq_restore(flags);
  6075. + spin_unlock_irqrestore(&up->port.lock, flags);
  6076. }
  6077. static int __init
  6078. diff -Nur linux-4.6.6.orig/drivers/tty/serial/sc16is7xx.c linux-4.6.6/drivers/tty/serial/sc16is7xx.c
  6079. --- linux-4.6.6.orig/drivers/tty/serial/sc16is7xx.c 2016-08-10 12:54:57.000000000 +0200
  6080. +++ linux-4.6.6/drivers/tty/serial/sc16is7xx.c 2016-08-22 21:27:47.281884646 +0200
  6081. @@ -1251,7 +1251,7 @@
  6082. /* Setup interrupt */
  6083. ret = devm_request_irq(dev, irq, sc16is7xx_irq,
  6084. - IRQF_ONESHOT | flags, dev_name(dev), s);
  6085. + flags, dev_name(dev), s);
  6086. if (!ret)
  6087. return 0;
  6088. diff -Nur linux-4.6.6.orig/drivers/usb/core/hcd.c linux-4.6.6/drivers/usb/core/hcd.c
  6089. --- linux-4.6.6.orig/drivers/usb/core/hcd.c 2016-08-10 12:54:57.000000000 +0200
  6090. +++ linux-4.6.6/drivers/usb/core/hcd.c 2016-08-22 21:27:47.281884646 +0200
  6091. @@ -1759,9 +1759,9 @@
  6092. * and no one may trigger the above deadlock situation when
  6093. * running complete() in tasklet.
  6094. */
  6095. - local_irq_save(flags);
  6096. + local_irq_save_nort(flags);
  6097. urb->complete(urb);
  6098. - local_irq_restore(flags);
  6099. + local_irq_restore_nort(flags);
  6100. usb_anchor_resume_wakeups(anchor);
  6101. atomic_dec(&urb->use_count);
  6102. diff -Nur linux-4.6.6.orig/drivers/usb/gadget/function/f_fs.c linux-4.6.6/drivers/usb/gadget/function/f_fs.c
  6103. --- linux-4.6.6.orig/drivers/usb/gadget/function/f_fs.c 2016-08-10 12:54:57.000000000 +0200
  6104. +++ linux-4.6.6/drivers/usb/gadget/function/f_fs.c 2016-08-22 21:27:47.285884906 +0200
  6105. @@ -1393,7 +1393,7 @@
  6106. pr_info("%s(): freeing\n", __func__);
  6107. ffs_data_clear(ffs);
  6108. BUG_ON(waitqueue_active(&ffs->ev.waitq) ||
  6109. - waitqueue_active(&ffs->ep0req_completion.wait));
  6110. + swait_active(&ffs->ep0req_completion.wait));
  6111. kfree(ffs->dev_name);
  6112. kfree(ffs);
  6113. }
  6114. diff -Nur linux-4.6.6.orig/drivers/usb/gadget/legacy/inode.c linux-4.6.6/drivers/usb/gadget/legacy/inode.c
  6115. --- linux-4.6.6.orig/drivers/usb/gadget/legacy/inode.c 2016-08-10 12:54:57.000000000 +0200
  6116. +++ linux-4.6.6/drivers/usb/gadget/legacy/inode.c 2016-08-22 21:27:47.285884906 +0200
  6117. @@ -346,7 +346,7 @@
  6118. spin_unlock_irq (&epdata->dev->lock);
  6119. if (likely (value == 0)) {
  6120. - value = wait_event_interruptible (done.wait, done.done);
  6121. + value = swait_event_interruptible (done.wait, done.done);
  6122. if (value != 0) {
  6123. spin_lock_irq (&epdata->dev->lock);
  6124. if (likely (epdata->ep != NULL)) {
  6125. @@ -355,7 +355,7 @@
  6126. usb_ep_dequeue (epdata->ep, epdata->req);
  6127. spin_unlock_irq (&epdata->dev->lock);
  6128. - wait_event (done.wait, done.done);
  6129. + swait_event (done.wait, done.done);
  6130. if (epdata->status == -ECONNRESET)
  6131. epdata->status = -EINTR;
  6132. } else {
  6133. diff -Nur linux-4.6.6.orig/drivers/usb/host/ohci-hcd.c linux-4.6.6/drivers/usb/host/ohci-hcd.c
  6134. --- linux-4.6.6.orig/drivers/usb/host/ohci-hcd.c 2016-08-10 12:54:57.000000000 +0200
  6135. +++ linux-4.6.6/drivers/usb/host/ohci-hcd.c 2016-08-22 21:27:47.289885159 +0200
  6136. @@ -500,7 +500,6 @@
  6137. setup_timer(&ohci->io_watchdog, io_watchdog_func,
  6138. (unsigned long) ohci);
  6139. - set_timer_slack(&ohci->io_watchdog, msecs_to_jiffies(20));
  6140. ohci->hcca = dma_alloc_coherent (hcd->self.controller,
  6141. sizeof(*ohci->hcca), &ohci->hcca_dma, GFP_KERNEL);
  6142. diff -Nur linux-4.6.6.orig/drivers/usb/host/xhci.c linux-4.6.6/drivers/usb/host/xhci.c
  6143. --- linux-4.6.6.orig/drivers/usb/host/xhci.c 2016-08-10 12:54:57.000000000 +0200
  6144. +++ linux-4.6.6/drivers/usb/host/xhci.c 2016-08-22 21:27:47.289885159 +0200
  6145. @@ -490,8 +490,6 @@
  6146. xhci->comp_mode_recovery_timer.expires = jiffies +
  6147. msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
  6148. - set_timer_slack(&xhci->comp_mode_recovery_timer,
  6149. - msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
  6150. add_timer(&xhci->comp_mode_recovery_timer);
  6151. xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
  6152. "Compliance mode recovery timer initialized");
  6153. diff -Nur linux-4.6.6.orig/fs/aio.c linux-4.6.6/fs/aio.c
  6154. --- linux-4.6.6.orig/fs/aio.c 2016-08-10 12:54:57.000000000 +0200
  6155. +++ linux-4.6.6/fs/aio.c 2016-08-22 21:27:47.293885419 +0200
  6156. @@ -40,6 +40,7 @@
  6157. #include <linux/ramfs.h>
  6158. #include <linux/percpu-refcount.h>
  6159. #include <linux/mount.h>
  6160. +#include <linux/swork.h>
  6161. #include <asm/kmap_types.h>
  6162. #include <asm/uaccess.h>
  6163. @@ -115,7 +116,7 @@
  6164. struct page **ring_pages;
  6165. long nr_pages;
  6166. - struct work_struct free_work;
  6167. + struct swork_event free_work;
  6168. /*
  6169. * signals when all in-flight requests are done
  6170. @@ -253,6 +254,7 @@
  6171. .mount = aio_mount,
  6172. .kill_sb = kill_anon_super,
  6173. };
  6174. + BUG_ON(swork_get());
  6175. aio_mnt = kern_mount(&aio_fs);
  6176. if (IS_ERR(aio_mnt))
  6177. panic("Failed to create aio fs mount.");
  6178. @@ -568,9 +570,9 @@
  6179. return cancel(&kiocb->common);
  6180. }
  6181. -static void free_ioctx(struct work_struct *work)
  6182. +static void free_ioctx(struct swork_event *sev)
  6183. {
  6184. - struct kioctx *ctx = container_of(work, struct kioctx, free_work);
  6185. + struct kioctx *ctx = container_of(sev, struct kioctx, free_work);
  6186. pr_debug("freeing %p\n", ctx);
  6187. @@ -589,8 +591,8 @@
  6188. if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
  6189. complete(&ctx->rq_wait->comp);
  6190. - INIT_WORK(&ctx->free_work, free_ioctx);
  6191. - schedule_work(&ctx->free_work);
  6192. + INIT_SWORK(&ctx->free_work, free_ioctx);
  6193. + swork_queue(&ctx->free_work);
  6194. }
  6195. /*
  6196. @@ -598,9 +600,9 @@
  6197. * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
  6198. * now it's safe to cancel any that need to be.
  6199. */
  6200. -static void free_ioctx_users(struct percpu_ref *ref)
  6201. +static void free_ioctx_users_work(struct swork_event *sev)
  6202. {
  6203. - struct kioctx *ctx = container_of(ref, struct kioctx, users);
  6204. + struct kioctx *ctx = container_of(sev, struct kioctx, free_work);
  6205. struct aio_kiocb *req;
  6206. spin_lock_irq(&ctx->ctx_lock);
  6207. @@ -619,6 +621,14 @@
  6208. percpu_ref_put(&ctx->reqs);
  6209. }
  6210. +static void free_ioctx_users(struct percpu_ref *ref)
  6211. +{
  6212. + struct kioctx *ctx = container_of(ref, struct kioctx, users);
  6213. +
  6214. + INIT_SWORK(&ctx->free_work, free_ioctx_users_work);
  6215. + swork_queue(&ctx->free_work);
  6216. +}
  6217. +
  6218. static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
  6219. {
  6220. unsigned i, new_nr;
  6221. diff -Nur linux-4.6.6.orig/fs/autofs4/autofs_i.h linux-4.6.6/fs/autofs4/autofs_i.h
  6222. --- linux-4.6.6.orig/fs/autofs4/autofs_i.h 2016-08-10 12:54:57.000000000 +0200
  6223. +++ linux-4.6.6/fs/autofs4/autofs_i.h 2016-08-22 21:27:47.293885419 +0200
  6224. @@ -30,6 +30,7 @@
  6225. #include <linux/sched.h>
  6226. #include <linux/mount.h>
  6227. #include <linux/namei.h>
  6228. +#include <linux/delay.h>
  6229. #include <asm/current.h>
  6230. #include <linux/uaccess.h>
  6231. diff -Nur linux-4.6.6.orig/fs/autofs4/expire.c linux-4.6.6/fs/autofs4/expire.c
  6232. --- linux-4.6.6.orig/fs/autofs4/expire.c 2016-08-10 12:54:57.000000000 +0200
  6233. +++ linux-4.6.6/fs/autofs4/expire.c 2016-08-22 21:27:47.293885419 +0200
  6234. @@ -148,7 +148,7 @@
  6235. parent = p->d_parent;
  6236. if (!spin_trylock(&parent->d_lock)) {
  6237. spin_unlock(&p->d_lock);
  6238. - cpu_relax();
  6239. + cpu_chill();
  6240. goto relock;
  6241. }
  6242. spin_unlock(&p->d_lock);
  6243. diff -Nur linux-4.6.6.orig/fs/buffer.c linux-4.6.6/fs/buffer.c
  6244. --- linux-4.6.6.orig/fs/buffer.c 2016-08-10 12:54:57.000000000 +0200
  6245. +++ linux-4.6.6/fs/buffer.c 2016-08-22 21:27:47.293885419 +0200
  6246. @@ -300,8 +300,7 @@
  6247. * decide that the page is now completely done.
  6248. */
  6249. first = page_buffers(page);
  6250. - local_irq_save(flags);
  6251. - bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  6252. + flags = bh_uptodate_lock_irqsave(first);
  6253. clear_buffer_async_read(bh);
  6254. unlock_buffer(bh);
  6255. tmp = bh;
  6256. @@ -314,8 +313,7 @@
  6257. }
  6258. tmp = tmp->b_this_page;
  6259. } while (tmp != bh);
  6260. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  6261. - local_irq_restore(flags);
  6262. + bh_uptodate_unlock_irqrestore(first, flags);
  6263. /*
  6264. * If none of the buffers had errors and they are all
  6265. @@ -327,9 +325,7 @@
  6266. return;
  6267. still_busy:
  6268. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  6269. - local_irq_restore(flags);
  6270. - return;
  6271. + bh_uptodate_unlock_irqrestore(first, flags);
  6272. }
  6273. /*
  6274. @@ -357,8 +353,7 @@
  6275. }
  6276. first = page_buffers(page);
  6277. - local_irq_save(flags);
  6278. - bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  6279. + flags = bh_uptodate_lock_irqsave(first);
  6280. clear_buffer_async_write(bh);
  6281. unlock_buffer(bh);
  6282. @@ -370,15 +365,12 @@
  6283. }
  6284. tmp = tmp->b_this_page;
  6285. }
  6286. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  6287. - local_irq_restore(flags);
  6288. + bh_uptodate_unlock_irqrestore(first, flags);
  6289. end_page_writeback(page);
  6290. return;
  6291. still_busy:
  6292. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  6293. - local_irq_restore(flags);
  6294. - return;
  6295. + bh_uptodate_unlock_irqrestore(first, flags);
  6296. }
  6297. EXPORT_SYMBOL(end_buffer_async_write);
  6298. @@ -3314,6 +3306,7 @@
  6299. struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
  6300. if (ret) {
  6301. INIT_LIST_HEAD(&ret->b_assoc_buffers);
  6302. + buffer_head_init_locks(ret);
  6303. preempt_disable();
  6304. __this_cpu_inc(bh_accounting.nr);
  6305. recalc_bh_state();
  6306. diff -Nur linux-4.6.6.orig/fs/dcache.c linux-4.6.6/fs/dcache.c
  6307. --- linux-4.6.6.orig/fs/dcache.c 2016-08-10 12:54:57.000000000 +0200
  6308. +++ linux-4.6.6/fs/dcache.c 2016-08-22 21:27:47.297885672 +0200
  6309. @@ -19,6 +19,7 @@
  6310. #include <linux/mm.h>
  6311. #include <linux/fs.h>
  6312. #include <linux/fsnotify.h>
  6313. +#include <linux/delay.h>
  6314. #include <linux/slab.h>
  6315. #include <linux/init.h>
  6316. #include <linux/hash.h>
  6317. @@ -578,7 +579,7 @@
  6318. failed:
  6319. spin_unlock(&dentry->d_lock);
  6320. - cpu_relax();
  6321. + cpu_chill();
  6322. return dentry; /* try again with same dentry */
  6323. }
  6324. @@ -2316,7 +2317,7 @@
  6325. if (dentry->d_lockref.count == 1) {
  6326. if (!spin_trylock(&inode->i_lock)) {
  6327. spin_unlock(&dentry->d_lock);
  6328. - cpu_relax();
  6329. + cpu_chill();
  6330. goto again;
  6331. }
  6332. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  6333. diff -Nur linux-4.6.6.orig/fs/eventpoll.c linux-4.6.6/fs/eventpoll.c
  6334. --- linux-4.6.6.orig/fs/eventpoll.c 2016-08-10 12:54:57.000000000 +0200
  6335. +++ linux-4.6.6/fs/eventpoll.c 2016-08-22 21:27:47.297885672 +0200
  6336. @@ -510,12 +510,12 @@
  6337. */
  6338. static void ep_poll_safewake(wait_queue_head_t *wq)
  6339. {
  6340. - int this_cpu = get_cpu();
  6341. + int this_cpu = get_cpu_light();
  6342. ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS,
  6343. ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu);
  6344. - put_cpu();
  6345. + put_cpu_light();
  6346. }
  6347. static void ep_remove_wait_queue(struct eppoll_entry *pwq)
  6348. diff -Nur linux-4.6.6.orig/fs/exec.c linux-4.6.6/fs/exec.c
  6349. --- linux-4.6.6.orig/fs/exec.c 2016-08-10 12:54:57.000000000 +0200
  6350. +++ linux-4.6.6/fs/exec.c 2016-08-22 21:27:47.301885933 +0200
  6351. @@ -961,12 +961,14 @@
  6352. }
  6353. }
  6354. task_lock(tsk);
  6355. + preempt_disable_rt();
  6356. active_mm = tsk->active_mm;
  6357. tsk->mm = mm;
  6358. tsk->active_mm = mm;
  6359. activate_mm(active_mm, mm);
  6360. tsk->mm->vmacache_seqnum = 0;
  6361. vmacache_flush(tsk);
  6362. + preempt_enable_rt();
  6363. task_unlock(tsk);
  6364. if (old_mm) {
  6365. up_read(&old_mm->mmap_sem);
  6366. diff -Nur linux-4.6.6.orig/fs/jbd2/checkpoint.c linux-4.6.6/fs/jbd2/checkpoint.c
  6367. --- linux-4.6.6.orig/fs/jbd2/checkpoint.c 2016-08-10 12:54:57.000000000 +0200
  6368. +++ linux-4.6.6/fs/jbd2/checkpoint.c 2016-08-22 21:27:47.301885933 +0200
  6369. @@ -116,6 +116,8 @@
  6370. nblocks = jbd2_space_needed(journal);
  6371. while (jbd2_log_space_left(journal) < nblocks) {
  6372. write_unlock(&journal->j_state_lock);
  6373. + if (current->plug)
  6374. + io_schedule();
  6375. mutex_lock(&journal->j_checkpoint_mutex);
  6376. /*
  6377. diff -Nur linux-4.6.6.orig/fs/namespace.c linux-4.6.6/fs/namespace.c
  6378. --- linux-4.6.6.orig/fs/namespace.c 2016-08-10 12:54:57.000000000 +0200
  6379. +++ linux-4.6.6/fs/namespace.c 2016-08-22 21:27:47.301885933 +0200
  6380. @@ -14,6 +14,7 @@
  6381. #include <linux/mnt_namespace.h>
  6382. #include <linux/user_namespace.h>
  6383. #include <linux/namei.h>
  6384. +#include <linux/delay.h>
  6385. #include <linux/security.h>
  6386. #include <linux/idr.h>
  6387. #include <linux/init.h> /* init_rootfs */
  6388. @@ -353,8 +354,11 @@
  6389. * incremented count after it has set MNT_WRITE_HOLD.
  6390. */
  6391. smp_mb();
  6392. - while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD)
  6393. - cpu_relax();
  6394. + while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) {
  6395. + preempt_enable();
  6396. + cpu_chill();
  6397. + preempt_disable();
  6398. + }
  6399. /*
  6400. * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will
  6401. * be set to match its requirements. So we must not load that until
  6402. diff -Nur linux-4.6.6.orig/fs/ntfs/aops.c linux-4.6.6/fs/ntfs/aops.c
  6403. --- linux-4.6.6.orig/fs/ntfs/aops.c 2016-08-10 12:54:57.000000000 +0200
  6404. +++ linux-4.6.6/fs/ntfs/aops.c 2016-08-22 21:27:47.305886186 +0200
  6405. @@ -92,13 +92,13 @@
  6406. ofs = 0;
  6407. if (file_ofs < init_size)
  6408. ofs = init_size - file_ofs;
  6409. - local_irq_save(flags);
  6410. + local_irq_save_nort(flags);
  6411. kaddr = kmap_atomic(page);
  6412. memset(kaddr + bh_offset(bh) + ofs, 0,
  6413. bh->b_size - ofs);
  6414. flush_dcache_page(page);
  6415. kunmap_atomic(kaddr);
  6416. - local_irq_restore(flags);
  6417. + local_irq_restore_nort(flags);
  6418. }
  6419. } else {
  6420. clear_buffer_uptodate(bh);
  6421. @@ -107,8 +107,7 @@
  6422. "0x%llx.", (unsigned long long)bh->b_blocknr);
  6423. }
  6424. first = page_buffers(page);
  6425. - local_irq_save(flags);
  6426. - bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  6427. + flags = bh_uptodate_lock_irqsave(first);
  6428. clear_buffer_async_read(bh);
  6429. unlock_buffer(bh);
  6430. tmp = bh;
  6431. @@ -123,8 +122,7 @@
  6432. }
  6433. tmp = tmp->b_this_page;
  6434. } while (tmp != bh);
  6435. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  6436. - local_irq_restore(flags);
  6437. + bh_uptodate_unlock_irqrestore(first, flags);
  6438. /*
  6439. * If none of the buffers had errors then we can set the page uptodate,
  6440. * but we first have to perform the post read mst fixups, if the
  6441. @@ -145,13 +143,13 @@
  6442. recs = PAGE_SIZE / rec_size;
  6443. /* Should have been verified before we got here... */
  6444. BUG_ON(!recs);
  6445. - local_irq_save(flags);
  6446. + local_irq_save_nort(flags);
  6447. kaddr = kmap_atomic(page);
  6448. for (i = 0; i < recs; i++)
  6449. post_read_mst_fixup((NTFS_RECORD*)(kaddr +
  6450. i * rec_size), rec_size);
  6451. kunmap_atomic(kaddr);
  6452. - local_irq_restore(flags);
  6453. + local_irq_restore_nort(flags);
  6454. flush_dcache_page(page);
  6455. if (likely(page_uptodate && !PageError(page)))
  6456. SetPageUptodate(page);
  6457. @@ -159,9 +157,7 @@
  6458. unlock_page(page);
  6459. return;
  6460. still_busy:
  6461. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  6462. - local_irq_restore(flags);
  6463. - return;
  6464. + bh_uptodate_unlock_irqrestore(first, flags);
  6465. }
  6466. /**
  6467. diff -Nur linux-4.6.6.orig/fs/timerfd.c linux-4.6.6/fs/timerfd.c
  6468. --- linux-4.6.6.orig/fs/timerfd.c 2016-08-10 12:54:57.000000000 +0200
  6469. +++ linux-4.6.6/fs/timerfd.c 2016-08-22 21:27:47.305886186 +0200
  6470. @@ -450,7 +450,10 @@
  6471. break;
  6472. }
  6473. spin_unlock_irq(&ctx->wqh.lock);
  6474. - cpu_relax();
  6475. + if (isalarm(ctx))
  6476. + hrtimer_wait_for_timer(&ctx->t.alarm.timer);
  6477. + else
  6478. + hrtimer_wait_for_timer(&ctx->t.tmr);
  6479. }
  6480. /*
  6481. diff -Nur linux-4.6.6.orig/include/acpi/platform/aclinux.h linux-4.6.6/include/acpi/platform/aclinux.h
  6482. --- linux-4.6.6.orig/include/acpi/platform/aclinux.h 2016-08-10 12:54:57.000000000 +0200
  6483. +++ linux-4.6.6/include/acpi/platform/aclinux.h 2016-08-22 21:27:47.305886186 +0200
  6484. @@ -127,6 +127,7 @@
  6485. #define acpi_cache_t struct kmem_cache
  6486. #define acpi_spinlock spinlock_t *
  6487. +#define acpi_raw_spinlock raw_spinlock_t *
  6488. #define acpi_cpu_flags unsigned long
  6489. /* Use native linux version of acpi_os_allocate_zeroed */
  6490. @@ -145,6 +146,20 @@
  6491. #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_get_thread_id
  6492. #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_create_lock
  6493. +#define acpi_os_create_raw_lock(__handle) \
  6494. +({ \
  6495. + raw_spinlock_t *lock = ACPI_ALLOCATE(sizeof(*lock)); \
  6496. + \
  6497. + if (lock) { \
  6498. + *(__handle) = lock; \
  6499. + raw_spin_lock_init(*(__handle)); \
  6500. + } \
  6501. + lock ? AE_OK : AE_NO_MEMORY; \
  6502. + })
  6503. +
  6504. +#define acpi_os_delete_raw_lock(__handle) kfree(__handle)
  6505. +
  6506. +
  6507. /*
  6508. * OSL interfaces used by debugger/disassembler
  6509. */
  6510. diff -Nur linux-4.6.6.orig/include/asm-generic/bug.h linux-4.6.6/include/asm-generic/bug.h
  6511. --- linux-4.6.6.orig/include/asm-generic/bug.h 2016-08-10 12:54:57.000000000 +0200
  6512. +++ linux-4.6.6/include/asm-generic/bug.h 2016-08-22 21:27:47.305886186 +0200
  6513. @@ -215,6 +215,20 @@
  6514. # define WARN_ON_SMP(x) ({0;})
  6515. #endif
  6516. +#ifdef CONFIG_PREEMPT_RT_BASE
  6517. +# define BUG_ON_RT(c) BUG_ON(c)
  6518. +# define BUG_ON_NONRT(c) do { } while (0)
  6519. +# define WARN_ON_RT(condition) WARN_ON(condition)
  6520. +# define WARN_ON_NONRT(condition) do { } while (0)
  6521. +# define WARN_ON_ONCE_NONRT(condition) do { } while (0)
  6522. +#else
  6523. +# define BUG_ON_RT(c) do { } while (0)
  6524. +# define BUG_ON_NONRT(c) BUG_ON(c)
  6525. +# define WARN_ON_RT(condition) do { } while (0)
  6526. +# define WARN_ON_NONRT(condition) WARN_ON(condition)
  6527. +# define WARN_ON_ONCE_NONRT(condition) WARN_ON_ONCE(condition)
  6528. +#endif
  6529. +
  6530. #endif /* __ASSEMBLY__ */
  6531. #endif
  6532. diff -Nur linux-4.6.6.orig/include/asm-generic/preempt.h linux-4.6.6/include/asm-generic/preempt.h
  6533. --- linux-4.6.6.orig/include/asm-generic/preempt.h 2016-08-10 12:54:57.000000000 +0200
  6534. +++ linux-4.6.6/include/asm-generic/preempt.h 2016-08-22 21:27:47.305886186 +0200
  6535. @@ -7,10 +7,10 @@
  6536. static __always_inline int preempt_count(void)
  6537. {
  6538. - return current_thread_info()->preempt_count;
  6539. + return READ_ONCE(current_thread_info()->preempt_count);
  6540. }
  6541. -static __always_inline int *preempt_count_ptr(void)
  6542. +static __always_inline volatile int *preempt_count_ptr(void)
  6543. {
  6544. return &current_thread_info()->preempt_count;
  6545. }
  6546. diff -Nur linux-4.6.6.orig/include/linux/blkdev.h linux-4.6.6/include/linux/blkdev.h
  6547. --- linux-4.6.6.orig/include/linux/blkdev.h 2016-08-10 12:54:57.000000000 +0200
  6548. +++ linux-4.6.6/include/linux/blkdev.h 2016-08-22 21:27:47.309886446 +0200
  6549. @@ -90,6 +90,7 @@
  6550. struct list_head queuelist;
  6551. union {
  6552. struct call_single_data csd;
  6553. + struct work_struct work;
  6554. unsigned long fifo_time;
  6555. };
  6556. @@ -457,7 +458,7 @@
  6557. struct throtl_data *td;
  6558. #endif
  6559. struct rcu_head rcu_head;
  6560. - wait_queue_head_t mq_freeze_wq;
  6561. + struct swait_queue_head mq_freeze_wq;
  6562. struct percpu_ref q_usage_counter;
  6563. struct list_head all_q_node;
  6564. diff -Nur linux-4.6.6.orig/include/linux/blk-mq.h linux-4.6.6/include/linux/blk-mq.h
  6565. --- linux-4.6.6.orig/include/linux/blk-mq.h 2016-08-10 12:54:57.000000000 +0200
  6566. +++ linux-4.6.6/include/linux/blk-mq.h 2016-08-22 21:27:47.305886186 +0200
  6567. @@ -218,6 +218,7 @@
  6568. struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index);
  6569. struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int);
  6570. +void __blk_mq_complete_request_remote_work(struct work_struct *work);
  6571. int blk_mq_request_started(struct request *rq);
  6572. void blk_mq_start_request(struct request *rq);
  6573. diff -Nur linux-4.6.6.orig/include/linux/bottom_half.h linux-4.6.6/include/linux/bottom_half.h
  6574. --- linux-4.6.6.orig/include/linux/bottom_half.h 2016-08-10 12:54:57.000000000 +0200
  6575. +++ linux-4.6.6/include/linux/bottom_half.h 2016-08-22 21:27:47.309886446 +0200
  6576. @@ -3,6 +3,39 @@
  6577. #include <linux/preempt.h>
  6578. +#ifdef CONFIG_PREEMPT_RT_FULL
  6579. +
  6580. +extern void __local_bh_disable(void);
  6581. +extern void _local_bh_enable(void);
  6582. +extern void __local_bh_enable(void);
  6583. +
  6584. +static inline void local_bh_disable(void)
  6585. +{
  6586. + __local_bh_disable();
  6587. +}
  6588. +
  6589. +static inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  6590. +{
  6591. + __local_bh_disable();
  6592. +}
  6593. +
  6594. +static inline void local_bh_enable(void)
  6595. +{
  6596. + __local_bh_enable();
  6597. +}
  6598. +
  6599. +static inline void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
  6600. +{
  6601. + __local_bh_enable();
  6602. +}
  6603. +
  6604. +static inline void local_bh_enable_ip(unsigned long ip)
  6605. +{
  6606. + __local_bh_enable();
  6607. +}
  6608. +
  6609. +#else
  6610. +
  6611. #ifdef CONFIG_TRACE_IRQFLAGS
  6612. extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt);
  6613. #else
  6614. @@ -30,5 +63,6 @@
  6615. {
  6616. __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
  6617. }
  6618. +#endif
  6619. #endif /* _LINUX_BH_H */
  6620. diff -Nur linux-4.6.6.orig/include/linux/buffer_head.h linux-4.6.6/include/linux/buffer_head.h
  6621. --- linux-4.6.6.orig/include/linux/buffer_head.h 2016-08-10 12:54:57.000000000 +0200
  6622. +++ linux-4.6.6/include/linux/buffer_head.h 2016-08-22 21:27:47.309886446 +0200
  6623. @@ -75,8 +75,50 @@
  6624. struct address_space *b_assoc_map; /* mapping this buffer is
  6625. associated with */
  6626. atomic_t b_count; /* users using this buffer_head */
  6627. +#ifdef CONFIG_PREEMPT_RT_BASE
  6628. + spinlock_t b_uptodate_lock;
  6629. +#if IS_ENABLED(CONFIG_JBD2)
  6630. + spinlock_t b_state_lock;
  6631. + spinlock_t b_journal_head_lock;
  6632. +#endif
  6633. +#endif
  6634. };
  6635. +static inline unsigned long bh_uptodate_lock_irqsave(struct buffer_head *bh)
  6636. +{
  6637. + unsigned long flags;
  6638. +
  6639. +#ifndef CONFIG_PREEMPT_RT_BASE
  6640. + local_irq_save(flags);
  6641. + bit_spin_lock(BH_Uptodate_Lock, &bh->b_state);
  6642. +#else
  6643. + spin_lock_irqsave(&bh->b_uptodate_lock, flags);
  6644. +#endif
  6645. + return flags;
  6646. +}
  6647. +
  6648. +static inline void
  6649. +bh_uptodate_unlock_irqrestore(struct buffer_head *bh, unsigned long flags)
  6650. +{
  6651. +#ifndef CONFIG_PREEMPT_RT_BASE
  6652. + bit_spin_unlock(BH_Uptodate_Lock, &bh->b_state);
  6653. + local_irq_restore(flags);
  6654. +#else
  6655. + spin_unlock_irqrestore(&bh->b_uptodate_lock, flags);
  6656. +#endif
  6657. +}
  6658. +
  6659. +static inline void buffer_head_init_locks(struct buffer_head *bh)
  6660. +{
  6661. +#ifdef CONFIG_PREEMPT_RT_BASE
  6662. + spin_lock_init(&bh->b_uptodate_lock);
  6663. +#if IS_ENABLED(CONFIG_JBD2)
  6664. + spin_lock_init(&bh->b_state_lock);
  6665. + spin_lock_init(&bh->b_journal_head_lock);
  6666. +#endif
  6667. +#endif
  6668. +}
  6669. +
  6670. /*
  6671. * macro tricks to expand the set_buffer_foo(), clear_buffer_foo()
  6672. * and buffer_foo() functions.
  6673. diff -Nur linux-4.6.6.orig/include/linux/cgroup-defs.h linux-4.6.6/include/linux/cgroup-defs.h
  6674. --- linux-4.6.6.orig/include/linux/cgroup-defs.h 2016-08-10 12:54:57.000000000 +0200
  6675. +++ linux-4.6.6/include/linux/cgroup-defs.h 2016-08-22 21:27:47.309886446 +0200
  6676. @@ -16,6 +16,7 @@
  6677. #include <linux/percpu-refcount.h>
  6678. #include <linux/percpu-rwsem.h>
  6679. #include <linux/workqueue.h>
  6680. +#include <linux/swork.h>
  6681. #ifdef CONFIG_CGROUPS
  6682. @@ -137,6 +138,7 @@
  6683. /* percpu_ref killing and RCU release */
  6684. struct rcu_head rcu_head;
  6685. struct work_struct destroy_work;
  6686. + struct swork_event destroy_swork;
  6687. };
  6688. /*
  6689. diff -Nur linux-4.6.6.orig/include/linux/completion.h linux-4.6.6/include/linux/completion.h
  6690. --- linux-4.6.6.orig/include/linux/completion.h 2016-08-10 12:54:57.000000000 +0200
  6691. +++ linux-4.6.6/include/linux/completion.h 2016-08-22 21:27:47.309886446 +0200
  6692. @@ -7,8 +7,7 @@
  6693. * Atomic wait-for-completion handler data structures.
  6694. * See kernel/sched/completion.c for details.
  6695. */
  6696. -
  6697. -#include <linux/wait.h>
  6698. +#include <linux/swait.h>
  6699. /*
  6700. * struct completion - structure used to maintain state for a "completion"
  6701. @@ -24,11 +23,11 @@
  6702. */
  6703. struct completion {
  6704. unsigned int done;
  6705. - wait_queue_head_t wait;
  6706. + struct swait_queue_head wait;
  6707. };
  6708. #define COMPLETION_INITIALIZER(work) \
  6709. - { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) }
  6710. + { 0, __SWAIT_QUEUE_HEAD_INITIALIZER((work).wait) }
  6711. #define COMPLETION_INITIALIZER_ONSTACK(work) \
  6712. ({ init_completion(&work); work; })
  6713. @@ -73,7 +72,7 @@
  6714. static inline void init_completion(struct completion *x)
  6715. {
  6716. x->done = 0;
  6717. - init_waitqueue_head(&x->wait);
  6718. + init_swait_queue_head(&x->wait);
  6719. }
  6720. /**
  6721. diff -Nur linux-4.6.6.orig/include/linux/cpu.h linux-4.6.6/include/linux/cpu.h
  6722. --- linux-4.6.6.orig/include/linux/cpu.h 2016-08-10 12:54:57.000000000 +0200
  6723. +++ linux-4.6.6/include/linux/cpu.h 2016-08-22 21:27:47.309886446 +0200
  6724. @@ -230,6 +230,8 @@
  6725. extern void put_online_cpus(void);
  6726. extern void cpu_hotplug_disable(void);
  6727. extern void cpu_hotplug_enable(void);
  6728. +extern void pin_current_cpu(void);
  6729. +extern void unpin_current_cpu(void);
  6730. #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri)
  6731. #define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri)
  6732. #define register_hotcpu_notifier(nb) register_cpu_notifier(nb)
  6733. @@ -247,6 +249,8 @@
  6734. #define put_online_cpus() do { } while (0)
  6735. #define cpu_hotplug_disable() do { } while (0)
  6736. #define cpu_hotplug_enable() do { } while (0)
  6737. +static inline void pin_current_cpu(void) { }
  6738. +static inline void unpin_current_cpu(void) { }
  6739. #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
  6740. #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
  6741. /* These aren't inline functions due to a GCC bug. */
  6742. diff -Nur linux-4.6.6.orig/include/linux/delay.h linux-4.6.6/include/linux/delay.h
  6743. --- linux-4.6.6.orig/include/linux/delay.h 2016-08-10 12:54:57.000000000 +0200
  6744. +++ linux-4.6.6/include/linux/delay.h 2016-08-22 21:27:47.309886446 +0200
  6745. @@ -52,4 +52,10 @@
  6746. msleep(seconds * 1000);
  6747. }
  6748. +#ifdef CONFIG_PREEMPT_RT_FULL
  6749. +extern void cpu_chill(void);
  6750. +#else
  6751. +# define cpu_chill() cpu_relax()
  6752. +#endif
  6753. +
  6754. #endif /* defined(_LINUX_DELAY_H) */
  6755. diff -Nur linux-4.6.6.orig/include/linux/highmem.h linux-4.6.6/include/linux/highmem.h
  6756. --- linux-4.6.6.orig/include/linux/highmem.h 2016-08-10 12:54:57.000000000 +0200
  6757. +++ linux-4.6.6/include/linux/highmem.h 2016-08-22 21:27:47.309886446 +0200
  6758. @@ -7,6 +7,7 @@
  6759. #include <linux/mm.h>
  6760. #include <linux/uaccess.h>
  6761. #include <linux/hardirq.h>
  6762. +#include <linux/sched.h>
  6763. #include <asm/cacheflush.h>
  6764. @@ -65,7 +66,7 @@
  6765. static inline void *kmap_atomic(struct page *page)
  6766. {
  6767. - preempt_disable();
  6768. + preempt_disable_nort();
  6769. pagefault_disable();
  6770. return page_address(page);
  6771. }
  6772. @@ -74,7 +75,7 @@
  6773. static inline void __kunmap_atomic(void *addr)
  6774. {
  6775. pagefault_enable();
  6776. - preempt_enable();
  6777. + preempt_enable_nort();
  6778. }
  6779. #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn))
  6780. @@ -86,32 +87,51 @@
  6781. #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  6782. +#ifndef CONFIG_PREEMPT_RT_FULL
  6783. DECLARE_PER_CPU(int, __kmap_atomic_idx);
  6784. +#endif
  6785. static inline int kmap_atomic_idx_push(void)
  6786. {
  6787. +#ifndef CONFIG_PREEMPT_RT_FULL
  6788. int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1;
  6789. -#ifdef CONFIG_DEBUG_HIGHMEM
  6790. +# ifdef CONFIG_DEBUG_HIGHMEM
  6791. WARN_ON_ONCE(in_irq() && !irqs_disabled());
  6792. BUG_ON(idx >= KM_TYPE_NR);
  6793. -#endif
  6794. +# endif
  6795. return idx;
  6796. +#else
  6797. + current->kmap_idx++;
  6798. + BUG_ON(current->kmap_idx > KM_TYPE_NR);
  6799. + return current->kmap_idx - 1;
  6800. +#endif
  6801. }
  6802. static inline int kmap_atomic_idx(void)
  6803. {
  6804. +#ifndef CONFIG_PREEMPT_RT_FULL
  6805. return __this_cpu_read(__kmap_atomic_idx) - 1;
  6806. +#else
  6807. + return current->kmap_idx - 1;
  6808. +#endif
  6809. }
  6810. static inline void kmap_atomic_idx_pop(void)
  6811. {
  6812. -#ifdef CONFIG_DEBUG_HIGHMEM
  6813. +#ifndef CONFIG_PREEMPT_RT_FULL
  6814. +# ifdef CONFIG_DEBUG_HIGHMEM
  6815. int idx = __this_cpu_dec_return(__kmap_atomic_idx);
  6816. BUG_ON(idx < 0);
  6817. -#else
  6818. +# else
  6819. __this_cpu_dec(__kmap_atomic_idx);
  6820. +# endif
  6821. +#else
  6822. + current->kmap_idx--;
  6823. +# ifdef CONFIG_DEBUG_HIGHMEM
  6824. + BUG_ON(current->kmap_idx < 0);
  6825. +# endif
  6826. #endif
  6827. }
  6828. diff -Nur linux-4.6.6.orig/include/linux/hrtimer.h linux-4.6.6/include/linux/hrtimer.h
  6829. --- linux-4.6.6.orig/include/linux/hrtimer.h 2016-08-10 12:54:57.000000000 +0200
  6830. +++ linux-4.6.6/include/linux/hrtimer.h 2016-08-22 21:27:47.313886699 +0200
  6831. @@ -87,6 +87,9 @@
  6832. * @function: timer expiry callback function
  6833. * @base: pointer to the timer base (per cpu and per clock)
  6834. * @state: state information (See bit values above)
  6835. + * @cb_entry: list entry to defer timers from hardirq context
  6836. + * @irqsafe: timer can run in hardirq context
  6837. + * @praecox: timer expiry time if expired at the time of programming
  6838. * @is_rel: Set if the timer was armed relative
  6839. * @start_pid: timer statistics field to store the pid of the task which
  6840. * started the timer
  6841. @@ -103,6 +106,11 @@
  6842. enum hrtimer_restart (*function)(struct hrtimer *);
  6843. struct hrtimer_clock_base *base;
  6844. u8 state;
  6845. + struct list_head cb_entry;
  6846. + int irqsafe;
  6847. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  6848. + ktime_t praecox;
  6849. +#endif
  6850. u8 is_rel;
  6851. #ifdef CONFIG_TIMER_STATS
  6852. int start_pid;
  6853. @@ -123,11 +131,7 @@
  6854. struct task_struct *task;
  6855. };
  6856. -#ifdef CONFIG_64BIT
  6857. # define HRTIMER_CLOCK_BASE_ALIGN 64
  6858. -#else
  6859. -# define HRTIMER_CLOCK_BASE_ALIGN 32
  6860. -#endif
  6861. /**
  6862. * struct hrtimer_clock_base - the timer base for a specific clock
  6863. @@ -136,6 +140,7 @@
  6864. * timer to a base on another cpu.
  6865. * @clockid: clock id for per_cpu support
  6866. * @active: red black tree root node for the active timers
  6867. + * @expired: list head for deferred timers.
  6868. * @get_time: function to retrieve the current time of the clock
  6869. * @offset: offset of this clock to the monotonic base
  6870. */
  6871. @@ -144,6 +149,7 @@
  6872. int index;
  6873. clockid_t clockid;
  6874. struct timerqueue_head active;
  6875. + struct list_head expired;
  6876. ktime_t (*get_time)(void);
  6877. ktime_t offset;
  6878. } __attribute__((__aligned__(HRTIMER_CLOCK_BASE_ALIGN)));
  6879. @@ -187,6 +193,7 @@
  6880. raw_spinlock_t lock;
  6881. seqcount_t seq;
  6882. struct hrtimer *running;
  6883. + struct hrtimer *running_soft;
  6884. unsigned int cpu;
  6885. unsigned int active_bases;
  6886. unsigned int clock_was_set_seq;
  6887. @@ -203,6 +210,9 @@
  6888. unsigned int nr_hangs;
  6889. unsigned int max_hang_time;
  6890. #endif
  6891. +#ifdef CONFIG_PREEMPT_RT_BASE
  6892. + wait_queue_head_t wait;
  6893. +#endif
  6894. struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
  6895. } ____cacheline_aligned;
  6896. @@ -412,6 +422,13 @@
  6897. hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
  6898. }
  6899. +/* Softirq preemption could deadlock timer removal */
  6900. +#ifdef CONFIG_PREEMPT_RT_BASE
  6901. + extern void hrtimer_wait_for_timer(const struct hrtimer *timer);
  6902. +#else
  6903. +# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0)
  6904. +#endif
  6905. +
  6906. /* Query timers: */
  6907. extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
  6908. @@ -436,7 +453,7 @@
  6909. * Helper function to check, whether the timer is running the callback
  6910. * function
  6911. */
  6912. -static inline int hrtimer_callback_running(struct hrtimer *timer)
  6913. +static inline int hrtimer_callback_running(const struct hrtimer *timer)
  6914. {
  6915. return timer->base->cpu_base->running == timer;
  6916. }
  6917. diff -Nur linux-4.6.6.orig/include/linux/idr.h linux-4.6.6/include/linux/idr.h
  6918. --- linux-4.6.6.orig/include/linux/idr.h 2016-08-10 12:54:57.000000000 +0200
  6919. +++ linux-4.6.6/include/linux/idr.h 2016-08-22 21:27:47.313886699 +0200
  6920. @@ -95,10 +95,14 @@
  6921. * Each idr_preload() should be matched with an invocation of this
  6922. * function. See idr_preload() for details.
  6923. */
  6924. +#ifdef CONFIG_PREEMPT_RT_FULL
  6925. +void idr_preload_end(void);
  6926. +#else
  6927. static inline void idr_preload_end(void)
  6928. {
  6929. preempt_enable();
  6930. }
  6931. +#endif
  6932. /**
  6933. * idr_find - return pointer for given id
  6934. diff -Nur linux-4.6.6.orig/include/linux/init_task.h linux-4.6.6/include/linux/init_task.h
  6935. --- linux-4.6.6.orig/include/linux/init_task.h 2016-08-10 12:54:57.000000000 +0200
  6936. +++ linux-4.6.6/include/linux/init_task.h 2016-08-22 21:27:47.313886699 +0200
  6937. @@ -148,6 +148,12 @@
  6938. # define INIT_PERF_EVENTS(tsk)
  6939. #endif
  6940. +#ifdef CONFIG_PREEMPT_RT_BASE
  6941. +# define INIT_TIMER_LIST .posix_timer_list = NULL,
  6942. +#else
  6943. +# define INIT_TIMER_LIST
  6944. +#endif
  6945. +
  6946. #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
  6947. # define INIT_VTIME(tsk) \
  6948. .vtime_seqcount = SEQCNT_ZERO(tsk.vtime_seqcount), \
  6949. @@ -239,6 +245,7 @@
  6950. .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \
  6951. .pi_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \
  6952. .timer_slack_ns = 50000, /* 50 usec default slack */ \
  6953. + INIT_TIMER_LIST \
  6954. .pids = { \
  6955. [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \
  6956. [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \
  6957. diff -Nur linux-4.6.6.orig/include/linux/interrupt.h linux-4.6.6/include/linux/interrupt.h
  6958. --- linux-4.6.6.orig/include/linux/interrupt.h 2016-08-10 12:54:57.000000000 +0200
  6959. +++ linux-4.6.6/include/linux/interrupt.h 2016-08-22 21:27:47.313886699 +0200
  6960. @@ -61,6 +61,7 @@
  6961. * interrupt handler after suspending interrupts. For system
  6962. * wakeup devices users need to implement wakeup detection in
  6963. * their interrupt handlers.
  6964. + * IRQF_NO_SOFTIRQ_CALL - Do not process softirqs in the irq thread context (RT)
  6965. */
  6966. #define IRQF_SHARED 0x00000080
  6967. #define IRQF_PROBE_SHARED 0x00000100
  6968. @@ -74,6 +75,7 @@
  6969. #define IRQF_NO_THREAD 0x00010000
  6970. #define IRQF_EARLY_RESUME 0x00020000
  6971. #define IRQF_COND_SUSPEND 0x00040000
  6972. +#define IRQF_NO_SOFTIRQ_CALL 0x00080000
  6973. #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD)
  6974. @@ -196,7 +198,7 @@
  6975. #ifdef CONFIG_LOCKDEP
  6976. # define local_irq_enable_in_hardirq() do { } while (0)
  6977. #else
  6978. -# define local_irq_enable_in_hardirq() local_irq_enable()
  6979. +# define local_irq_enable_in_hardirq() local_irq_enable_nort()
  6980. #endif
  6981. extern void disable_irq_nosync(unsigned int irq);
  6982. @@ -217,6 +219,7 @@
  6983. * @irq: Interrupt to which notification applies
  6984. * @kref: Reference count, for internal use
  6985. * @work: Work item, for internal use
  6986. + * @list: List item for deferred callbacks
  6987. * @notify: Function to be called on change. This will be
  6988. * called in process context.
  6989. * @release: Function to be called on release. This will be
  6990. @@ -228,6 +231,7 @@
  6991. unsigned int irq;
  6992. struct kref kref;
  6993. struct work_struct work;
  6994. + struct list_head list;
  6995. void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask);
  6996. void (*release)(struct kref *ref);
  6997. };
  6998. @@ -390,9 +394,13 @@
  6999. bool state);
  7000. #ifdef CONFIG_IRQ_FORCED_THREADING
  7001. +# ifndef CONFIG_PREEMPT_RT_BASE
  7002. extern bool force_irqthreads;
  7003. +# else
  7004. +# define force_irqthreads (true)
  7005. +# endif
  7006. #else
  7007. -#define force_irqthreads (0)
  7008. +#define force_irqthreads (false)
  7009. #endif
  7010. #ifndef __ARCH_SET_SOFTIRQ_PENDING
  7011. @@ -449,9 +457,10 @@
  7012. void (*action)(struct softirq_action *);
  7013. };
  7014. +#ifndef CONFIG_PREEMPT_RT_FULL
  7015. asmlinkage void do_softirq(void);
  7016. asmlinkage void __do_softirq(void);
  7017. -
  7018. +static inline void thread_do_softirq(void) { do_softirq(); }
  7019. #ifdef __ARCH_HAS_DO_SOFTIRQ
  7020. void do_softirq_own_stack(void);
  7021. #else
  7022. @@ -460,13 +469,25 @@
  7023. __do_softirq();
  7024. }
  7025. #endif
  7026. +#else
  7027. +extern void thread_do_softirq(void);
  7028. +#endif
  7029. extern void open_softirq(int nr, void (*action)(struct softirq_action *));
  7030. extern void softirq_init(void);
  7031. extern void __raise_softirq_irqoff(unsigned int nr);
  7032. +#ifdef CONFIG_PREEMPT_RT_FULL
  7033. +extern void __raise_softirq_irqoff_ksoft(unsigned int nr);
  7034. +#else
  7035. +static inline void __raise_softirq_irqoff_ksoft(unsigned int nr)
  7036. +{
  7037. + __raise_softirq_irqoff(nr);
  7038. +}
  7039. +#endif
  7040. extern void raise_softirq_irqoff(unsigned int nr);
  7041. extern void raise_softirq(unsigned int nr);
  7042. +extern void softirq_check_pending_idle(void);
  7043. DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
  7044. @@ -488,8 +509,9 @@
  7045. to be executed on some cpu at least once after this.
  7046. * If the tasklet is already scheduled, but its execution is still not
  7047. started, it will be executed only once.
  7048. - * If this tasklet is already running on another CPU (or schedule is called
  7049. - from tasklet itself), it is rescheduled for later.
  7050. + * If this tasklet is already running on another CPU, it is rescheduled
  7051. + for later.
  7052. + * Schedule must not be called from the tasklet itself (a lockup occurs)
  7053. * Tasklet is strictly serialized wrt itself, but not
  7054. wrt another tasklets. If client needs some intertask synchronization,
  7055. he makes it with spinlocks.
  7056. @@ -514,27 +536,36 @@
  7057. enum
  7058. {
  7059. TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */
  7060. - TASKLET_STATE_RUN /* Tasklet is running (SMP only) */
  7061. + TASKLET_STATE_RUN, /* Tasklet is running (SMP only) */
  7062. + TASKLET_STATE_PENDING /* Tasklet is pending */
  7063. };
  7064. -#ifdef CONFIG_SMP
  7065. +#define TASKLET_STATEF_SCHED (1 << TASKLET_STATE_SCHED)
  7066. +#define TASKLET_STATEF_RUN (1 << TASKLET_STATE_RUN)
  7067. +#define TASKLET_STATEF_PENDING (1 << TASKLET_STATE_PENDING)
  7068. +
  7069. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  7070. static inline int tasklet_trylock(struct tasklet_struct *t)
  7071. {
  7072. return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state);
  7073. }
  7074. +static inline int tasklet_tryunlock(struct tasklet_struct *t)
  7075. +{
  7076. + return cmpxchg(&t->state, TASKLET_STATEF_RUN, 0) == TASKLET_STATEF_RUN;
  7077. +}
  7078. +
  7079. static inline void tasklet_unlock(struct tasklet_struct *t)
  7080. {
  7081. smp_mb__before_atomic();
  7082. clear_bit(TASKLET_STATE_RUN, &(t)->state);
  7083. }
  7084. -static inline void tasklet_unlock_wait(struct tasklet_struct *t)
  7085. -{
  7086. - while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); }
  7087. -}
  7088. +extern void tasklet_unlock_wait(struct tasklet_struct *t);
  7089. +
  7090. #else
  7091. #define tasklet_trylock(t) 1
  7092. +#define tasklet_tryunlock(t) 1
  7093. #define tasklet_unlock_wait(t) do { } while (0)
  7094. #define tasklet_unlock(t) do { } while (0)
  7095. #endif
  7096. @@ -583,12 +614,7 @@
  7097. smp_mb();
  7098. }
  7099. -static inline void tasklet_enable(struct tasklet_struct *t)
  7100. -{
  7101. - smp_mb__before_atomic();
  7102. - atomic_dec(&t->count);
  7103. -}
  7104. -
  7105. +extern void tasklet_enable(struct tasklet_struct *t);
  7106. extern void tasklet_kill(struct tasklet_struct *t);
  7107. extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu);
  7108. extern void tasklet_init(struct tasklet_struct *t,
  7109. @@ -619,6 +645,12 @@
  7110. tasklet_kill(&ttimer->tasklet);
  7111. }
  7112. +#ifdef CONFIG_PREEMPT_RT_FULL
  7113. +extern void softirq_early_init(void);
  7114. +#else
  7115. +static inline void softirq_early_init(void) { }
  7116. +#endif
  7117. +
  7118. /*
  7119. * Autoprobing for irqs:
  7120. *
  7121. diff -Nur linux-4.6.6.orig/include/linux/irqdesc.h linux-4.6.6/include/linux/irqdesc.h
  7122. --- linux-4.6.6.orig/include/linux/irqdesc.h 2016-08-10 12:54:57.000000000 +0200
  7123. +++ linux-4.6.6/include/linux/irqdesc.h 2016-08-22 21:27:47.313886699 +0200
  7124. @@ -64,6 +64,7 @@
  7125. unsigned int irqs_unhandled;
  7126. atomic_t threads_handled;
  7127. int threads_handled_last;
  7128. + u64 random_ip;
  7129. raw_spinlock_t lock;
  7130. struct cpumask *percpu_enabled;
  7131. #ifdef CONFIG_SMP
  7132. diff -Nur linux-4.6.6.orig/include/linux/irqflags.h linux-4.6.6/include/linux/irqflags.h
  7133. --- linux-4.6.6.orig/include/linux/irqflags.h 2016-08-10 12:54:57.000000000 +0200
  7134. +++ linux-4.6.6/include/linux/irqflags.h 2016-08-22 21:27:47.317886959 +0200
  7135. @@ -25,8 +25,6 @@
  7136. # define trace_softirqs_enabled(p) ((p)->softirqs_enabled)
  7137. # define trace_hardirq_enter() do { current->hardirq_context++; } while (0)
  7138. # define trace_hardirq_exit() do { current->hardirq_context--; } while (0)
  7139. -# define lockdep_softirq_enter() do { current->softirq_context++; } while (0)
  7140. -# define lockdep_softirq_exit() do { current->softirq_context--; } while (0)
  7141. # define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1,
  7142. #else
  7143. # define trace_hardirqs_on() do { } while (0)
  7144. @@ -39,9 +37,15 @@
  7145. # define trace_softirqs_enabled(p) 0
  7146. # define trace_hardirq_enter() do { } while (0)
  7147. # define trace_hardirq_exit() do { } while (0)
  7148. +# define INIT_TRACE_IRQFLAGS
  7149. +#endif
  7150. +
  7151. +#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL)
  7152. +# define lockdep_softirq_enter() do { current->softirq_context++; } while (0)
  7153. +# define lockdep_softirq_exit() do { current->softirq_context--; } while (0)
  7154. +#else
  7155. # define lockdep_softirq_enter() do { } while (0)
  7156. # define lockdep_softirq_exit() do { } while (0)
  7157. -# define INIT_TRACE_IRQFLAGS
  7158. #endif
  7159. #if defined(CONFIG_IRQSOFF_TRACER) || \
  7160. @@ -148,4 +152,23 @@
  7161. #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags)
  7162. +/*
  7163. + * local_irq* variants depending on RT/!RT
  7164. + */
  7165. +#ifdef CONFIG_PREEMPT_RT_FULL
  7166. +# define local_irq_disable_nort() do { } while (0)
  7167. +# define local_irq_enable_nort() do { } while (0)
  7168. +# define local_irq_save_nort(flags) local_save_flags(flags)
  7169. +# define local_irq_restore_nort(flags) (void)(flags)
  7170. +# define local_irq_disable_rt() local_irq_disable()
  7171. +# define local_irq_enable_rt() local_irq_enable()
  7172. +#else
  7173. +# define local_irq_disable_nort() local_irq_disable()
  7174. +# define local_irq_enable_nort() local_irq_enable()
  7175. +# define local_irq_save_nort(flags) local_irq_save(flags)
  7176. +# define local_irq_restore_nort(flags) local_irq_restore(flags)
  7177. +# define local_irq_disable_rt() do { } while (0)
  7178. +# define local_irq_enable_rt() do { } while (0)
  7179. +#endif
  7180. +
  7181. #endif
  7182. diff -Nur linux-4.6.6.orig/include/linux/irq.h linux-4.6.6/include/linux/irq.h
  7183. --- linux-4.6.6.orig/include/linux/irq.h 2016-08-10 12:54:57.000000000 +0200
  7184. +++ linux-4.6.6/include/linux/irq.h 2016-08-22 21:27:47.313886699 +0200
  7185. @@ -72,6 +72,7 @@
  7186. * IRQ_IS_POLLED - Always polled by another interrupt. Exclude
  7187. * it from the spurious interrupt detection
  7188. * mechanism and from core side polling.
  7189. + * IRQ_NO_SOFTIRQ_CALL - No softirq processing in the irq thread context (RT)
  7190. * IRQ_DISABLE_UNLAZY - Disable lazy irq disable
  7191. */
  7192. enum {
  7193. @@ -99,13 +100,14 @@
  7194. IRQ_PER_CPU_DEVID = (1 << 17),
  7195. IRQ_IS_POLLED = (1 << 18),
  7196. IRQ_DISABLE_UNLAZY = (1 << 19),
  7197. + IRQ_NO_SOFTIRQ_CALL = (1 << 20),
  7198. };
  7199. #define IRQF_MODIFY_MASK \
  7200. (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \
  7201. IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \
  7202. IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \
  7203. - IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY)
  7204. + IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY | IRQ_NO_SOFTIRQ_CALL)
  7205. #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING)
  7206. diff -Nur linux-4.6.6.orig/include/linux/irq_work.h linux-4.6.6/include/linux/irq_work.h
  7207. --- linux-4.6.6.orig/include/linux/irq_work.h 2016-08-10 12:54:57.000000000 +0200
  7208. +++ linux-4.6.6/include/linux/irq_work.h 2016-08-22 21:27:47.313886699 +0200
  7209. @@ -16,6 +16,7 @@
  7210. #define IRQ_WORK_BUSY 2UL
  7211. #define IRQ_WORK_FLAGS 3UL
  7212. #define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */
  7213. +#define IRQ_WORK_HARD_IRQ 8UL /* Run hard IRQ context, even on RT */
  7214. struct irq_work {
  7215. unsigned long flags;
  7216. @@ -51,4 +52,10 @@
  7217. static inline void irq_work_run(void) { }
  7218. #endif
  7219. +#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL)
  7220. +void irq_work_tick_soft(void);
  7221. +#else
  7222. +static inline void irq_work_tick_soft(void) { }
  7223. +#endif
  7224. +
  7225. #endif /* _LINUX_IRQ_WORK_H */
  7226. diff -Nur linux-4.6.6.orig/include/linux/jbd2.h linux-4.6.6/include/linux/jbd2.h
  7227. --- linux-4.6.6.orig/include/linux/jbd2.h 2016-08-10 12:54:57.000000000 +0200
  7228. +++ linux-4.6.6/include/linux/jbd2.h 2016-08-22 21:27:47.317886959 +0200
  7229. @@ -347,32 +347,56 @@
  7230. static inline void jbd_lock_bh_state(struct buffer_head *bh)
  7231. {
  7232. +#ifndef CONFIG_PREEMPT_RT_BASE
  7233. bit_spin_lock(BH_State, &bh->b_state);
  7234. +#else
  7235. + spin_lock(&bh->b_state_lock);
  7236. +#endif
  7237. }
  7238. static inline int jbd_trylock_bh_state(struct buffer_head *bh)
  7239. {
  7240. +#ifndef CONFIG_PREEMPT_RT_BASE
  7241. return bit_spin_trylock(BH_State, &bh->b_state);
  7242. +#else
  7243. + return spin_trylock(&bh->b_state_lock);
  7244. +#endif
  7245. }
  7246. static inline int jbd_is_locked_bh_state(struct buffer_head *bh)
  7247. {
  7248. +#ifndef CONFIG_PREEMPT_RT_BASE
  7249. return bit_spin_is_locked(BH_State, &bh->b_state);
  7250. +#else
  7251. + return spin_is_locked(&bh->b_state_lock);
  7252. +#endif
  7253. }
  7254. static inline void jbd_unlock_bh_state(struct buffer_head *bh)
  7255. {
  7256. +#ifndef CONFIG_PREEMPT_RT_BASE
  7257. bit_spin_unlock(BH_State, &bh->b_state);
  7258. +#else
  7259. + spin_unlock(&bh->b_state_lock);
  7260. +#endif
  7261. }
  7262. static inline void jbd_lock_bh_journal_head(struct buffer_head *bh)
  7263. {
  7264. +#ifndef CONFIG_PREEMPT_RT_BASE
  7265. bit_spin_lock(BH_JournalHead, &bh->b_state);
  7266. +#else
  7267. + spin_lock(&bh->b_journal_head_lock);
  7268. +#endif
  7269. }
  7270. static inline void jbd_unlock_bh_journal_head(struct buffer_head *bh)
  7271. {
  7272. +#ifndef CONFIG_PREEMPT_RT_BASE
  7273. bit_spin_unlock(BH_JournalHead, &bh->b_state);
  7274. +#else
  7275. + spin_unlock(&bh->b_journal_head_lock);
  7276. +#endif
  7277. }
  7278. #define J_ASSERT(assert) BUG_ON(!(assert))
  7279. diff -Nur linux-4.6.6.orig/include/linux/kdb.h linux-4.6.6/include/linux/kdb.h
  7280. --- linux-4.6.6.orig/include/linux/kdb.h 2016-08-10 12:54:57.000000000 +0200
  7281. +++ linux-4.6.6/include/linux/kdb.h 2016-08-22 21:27:47.317886959 +0200
  7282. @@ -167,6 +167,7 @@
  7283. extern __printf(1, 2) int kdb_printf(const char *, ...);
  7284. typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...);
  7285. +#define in_kdb_printk() (kdb_trap_printk)
  7286. extern void kdb_init(int level);
  7287. /* Access to kdb specific polling devices */
  7288. @@ -201,6 +202,7 @@
  7289. extern int kdb_unregister(char *);
  7290. #else /* ! CONFIG_KGDB_KDB */
  7291. static inline __printf(1, 2) int kdb_printf(const char *fmt, ...) { return 0; }
  7292. +#define in_kdb_printk() (0)
  7293. static inline void kdb_init(int level) {}
  7294. static inline int kdb_register(char *cmd, kdb_func_t func, char *usage,
  7295. char *help, short minlen) { return 0; }
  7296. diff -Nur linux-4.6.6.orig/include/linux/kernel.h linux-4.6.6/include/linux/kernel.h
  7297. --- linux-4.6.6.orig/include/linux/kernel.h 2016-08-10 12:54:57.000000000 +0200
  7298. +++ linux-4.6.6/include/linux/kernel.h 2016-08-22 21:27:47.317886959 +0200
  7299. @@ -188,6 +188,9 @@
  7300. */
  7301. # define might_sleep() \
  7302. do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0)
  7303. +
  7304. +# define might_sleep_no_state_check() \
  7305. + do { ___might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0)
  7306. # define sched_annotate_sleep() (current->task_state_change = 0)
  7307. #else
  7308. static inline void ___might_sleep(const char *file, int line,
  7309. @@ -195,6 +198,7 @@
  7310. static inline void __might_sleep(const char *file, int line,
  7311. int preempt_offset) { }
  7312. # define might_sleep() do { might_resched(); } while (0)
  7313. +# define might_sleep_no_state_check() do { might_resched(); } while (0)
  7314. # define sched_annotate_sleep() do { } while (0)
  7315. #endif
  7316. @@ -484,6 +488,7 @@
  7317. SYSTEM_HALT,
  7318. SYSTEM_POWER_OFF,
  7319. SYSTEM_RESTART,
  7320. + SYSTEM_SUSPEND,
  7321. } system_state;
  7322. #define TAINT_PROPRIETARY_MODULE 0
  7323. diff -Nur linux-4.6.6.orig/include/linux/lglock.h linux-4.6.6/include/linux/lglock.h
  7324. --- linux-4.6.6.orig/include/linux/lglock.h 2016-08-10 12:54:57.000000000 +0200
  7325. +++ linux-4.6.6/include/linux/lglock.h 2016-08-22 21:27:47.317886959 +0200
  7326. @@ -34,13 +34,30 @@
  7327. #endif
  7328. struct lglock {
  7329. +#ifdef CONFIG_PREEMPT_RT_FULL
  7330. + struct rt_mutex __percpu *lock;
  7331. +#else
  7332. arch_spinlock_t __percpu *lock;
  7333. +#endif
  7334. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  7335. struct lock_class_key lock_key;
  7336. struct lockdep_map lock_dep_map;
  7337. #endif
  7338. };
  7339. +#ifdef CONFIG_PREEMPT_RT_FULL
  7340. +# define DEFINE_LGLOCK(name) \
  7341. + static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \
  7342. + = __RT_MUTEX_INITIALIZER( name ## _lock); \
  7343. + struct lglock name = { .lock = &name ## _lock }
  7344. +
  7345. +# define DEFINE_STATIC_LGLOCK(name) \
  7346. + static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \
  7347. + = __RT_MUTEX_INITIALIZER( name ## _lock); \
  7348. + static struct lglock name = { .lock = &name ## _lock }
  7349. +
  7350. +#else
  7351. +
  7352. #define DEFINE_LGLOCK(name) \
  7353. static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \
  7354. = __ARCH_SPIN_LOCK_UNLOCKED; \
  7355. @@ -50,6 +67,7 @@
  7356. static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \
  7357. = __ARCH_SPIN_LOCK_UNLOCKED; \
  7358. static struct lglock name = { .lock = &name ## _lock }
  7359. +#endif
  7360. void lg_lock_init(struct lglock *lg, char *name);
  7361. @@ -64,6 +82,12 @@
  7362. void lg_global_lock(struct lglock *lg);
  7363. void lg_global_unlock(struct lglock *lg);
  7364. +#ifndef CONFIG_PREEMPT_RT_FULL
  7365. +#define lg_global_trylock_relax(name) lg_global_lock(name)
  7366. +#else
  7367. +void lg_global_trylock_relax(struct lglock *lg);
  7368. +#endif
  7369. +
  7370. #else
  7371. /* When !CONFIG_SMP, map lglock to spinlock */
  7372. #define lglock spinlock
  7373. diff -Nur linux-4.6.6.orig/include/linux/list_bl.h linux-4.6.6/include/linux/list_bl.h
  7374. --- linux-4.6.6.orig/include/linux/list_bl.h 2016-08-10 12:54:57.000000000 +0200
  7375. +++ linux-4.6.6/include/linux/list_bl.h 2016-08-22 21:27:47.321887212 +0200
  7376. @@ -2,6 +2,7 @@
  7377. #define _LINUX_LIST_BL_H
  7378. #include <linux/list.h>
  7379. +#include <linux/spinlock.h>
  7380. #include <linux/bit_spinlock.h>
  7381. /*
  7382. @@ -32,13 +33,24 @@
  7383. struct hlist_bl_head {
  7384. struct hlist_bl_node *first;
  7385. +#ifdef CONFIG_PREEMPT_RT_BASE
  7386. + raw_spinlock_t lock;
  7387. +#endif
  7388. };
  7389. struct hlist_bl_node {
  7390. struct hlist_bl_node *next, **pprev;
  7391. };
  7392. -#define INIT_HLIST_BL_HEAD(ptr) \
  7393. - ((ptr)->first = NULL)
  7394. +
  7395. +#ifdef CONFIG_PREEMPT_RT_BASE
  7396. +#define INIT_HLIST_BL_HEAD(h) \
  7397. +do { \
  7398. + (h)->first = NULL; \
  7399. + raw_spin_lock_init(&(h)->lock); \
  7400. +} while (0)
  7401. +#else
  7402. +#define INIT_HLIST_BL_HEAD(h) (h)->first = NULL
  7403. +#endif
  7404. static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h)
  7405. {
  7406. @@ -118,12 +130,26 @@
  7407. static inline void hlist_bl_lock(struct hlist_bl_head *b)
  7408. {
  7409. +#ifndef CONFIG_PREEMPT_RT_BASE
  7410. bit_spin_lock(0, (unsigned long *)b);
  7411. +#else
  7412. + raw_spin_lock(&b->lock);
  7413. +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
  7414. + __set_bit(0, (unsigned long *)b);
  7415. +#endif
  7416. +#endif
  7417. }
  7418. static inline void hlist_bl_unlock(struct hlist_bl_head *b)
  7419. {
  7420. +#ifndef CONFIG_PREEMPT_RT_BASE
  7421. __bit_spin_unlock(0, (unsigned long *)b);
  7422. +#else
  7423. +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
  7424. + __clear_bit(0, (unsigned long *)b);
  7425. +#endif
  7426. + raw_spin_unlock(&b->lock);
  7427. +#endif
  7428. }
  7429. static inline bool hlist_bl_is_locked(struct hlist_bl_head *b)
  7430. diff -Nur linux-4.6.6.orig/include/linux/list.h linux-4.6.6/include/linux/list.h
  7431. --- linux-4.6.6.orig/include/linux/list.h 2016-08-10 12:54:57.000000000 +0200
  7432. +++ linux-4.6.6/include/linux/list.h 2016-08-22 21:27:47.317886959 +0200
  7433. @@ -679,6 +679,16 @@
  7434. }
  7435. /*
  7436. + * Check whether the node is the only node of the head without
  7437. + * accessing head.
  7438. + */
  7439. +static inline bool hlist_is_singular_node(struct hlist_node *n,
  7440. + struct hlist_head *h)
  7441. +{
  7442. + return !n->next && n->pprev == &h->first;
  7443. +}
  7444. +
  7445. +/*
  7446. * Move a list from one list head to another. Fixup the pprev
  7447. * reference of the first entry if it exists.
  7448. */
  7449. diff -Nur linux-4.6.6.orig/include/linux/locallock.h linux-4.6.6/include/linux/locallock.h
  7450. --- linux-4.6.6.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100
  7451. +++ linux-4.6.6/include/linux/locallock.h 2016-08-22 21:27:47.321887212 +0200
  7452. @@ -0,0 +1,278 @@
  7453. +#ifndef _LINUX_LOCALLOCK_H
  7454. +#define _LINUX_LOCALLOCK_H
  7455. +
  7456. +#include <linux/percpu.h>
  7457. +#include <linux/spinlock.h>
  7458. +
  7459. +#ifdef CONFIG_PREEMPT_RT_BASE
  7460. +
  7461. +#ifdef CONFIG_DEBUG_SPINLOCK
  7462. +# define LL_WARN(cond) WARN_ON(cond)
  7463. +#else
  7464. +# define LL_WARN(cond) do { } while (0)
  7465. +#endif
  7466. +
  7467. +/*
  7468. + * per cpu lock based substitute for local_irq_*()
  7469. + */
  7470. +struct local_irq_lock {
  7471. + spinlock_t lock;
  7472. + struct task_struct *owner;
  7473. + int nestcnt;
  7474. + unsigned long flags;
  7475. +};
  7476. +
  7477. +#define DEFINE_LOCAL_IRQ_LOCK(lvar) \
  7478. + DEFINE_PER_CPU(struct local_irq_lock, lvar) = { \
  7479. + .lock = __SPIN_LOCK_UNLOCKED((lvar).lock) }
  7480. +
  7481. +#define DECLARE_LOCAL_IRQ_LOCK(lvar) \
  7482. + DECLARE_PER_CPU(struct local_irq_lock, lvar)
  7483. +
  7484. +#define local_irq_lock_init(lvar) \
  7485. + do { \
  7486. + int __cpu; \
  7487. + for_each_possible_cpu(__cpu) \
  7488. + spin_lock_init(&per_cpu(lvar, __cpu).lock); \
  7489. + } while (0)
  7490. +
  7491. +/*
  7492. + * spin_lock|trylock|unlock_local flavour that does not migrate disable
  7493. + * used for __local_lock|trylock|unlock where get_local_var/put_local_var
  7494. + * already takes care of the migrate_disable/enable
  7495. + * for CONFIG_PREEMPT_BASE map to the normal spin_* calls.
  7496. + */
  7497. +#ifdef CONFIG_PREEMPT_RT_FULL
  7498. +# define spin_lock_local(lock) rt_spin_lock__no_mg(lock)
  7499. +# define spin_trylock_local(lock) rt_spin_trylock__no_mg(lock)
  7500. +# define spin_unlock_local(lock) rt_spin_unlock__no_mg(lock)
  7501. +#else
  7502. +# define spin_lock_local(lock) spin_lock(lock)
  7503. +# define spin_trylock_local(lock) spin_trylock(lock)
  7504. +# define spin_unlock_local(lock) spin_unlock(lock)
  7505. +#endif
  7506. +
  7507. +static inline void __local_lock(struct local_irq_lock *lv)
  7508. +{
  7509. + if (lv->owner != current) {
  7510. + spin_lock_local(&lv->lock);
  7511. + LL_WARN(lv->owner);
  7512. + LL_WARN(lv->nestcnt);
  7513. + lv->owner = current;
  7514. + }
  7515. + lv->nestcnt++;
  7516. +}
  7517. +
  7518. +#define local_lock(lvar) \
  7519. + do { __local_lock(&get_local_var(lvar)); } while (0)
  7520. +
  7521. +#define local_lock_on(lvar, cpu) \
  7522. + do { __local_lock(&per_cpu(lvar, cpu)); } while (0)
  7523. +
  7524. +static inline int __local_trylock(struct local_irq_lock *lv)
  7525. +{
  7526. + if (lv->owner != current && spin_trylock_local(&lv->lock)) {
  7527. + LL_WARN(lv->owner);
  7528. + LL_WARN(lv->nestcnt);
  7529. + lv->owner = current;
  7530. + lv->nestcnt = 1;
  7531. + return 1;
  7532. + }
  7533. + return 0;
  7534. +}
  7535. +
  7536. +#define local_trylock(lvar) \
  7537. + ({ \
  7538. + int __locked; \
  7539. + __locked = __local_trylock(&get_local_var(lvar)); \
  7540. + if (!__locked) \
  7541. + put_local_var(lvar); \
  7542. + __locked; \
  7543. + })
  7544. +
  7545. +static inline void __local_unlock(struct local_irq_lock *lv)
  7546. +{
  7547. + LL_WARN(lv->nestcnt == 0);
  7548. + LL_WARN(lv->owner != current);
  7549. + if (--lv->nestcnt)
  7550. + return;
  7551. +
  7552. + lv->owner = NULL;
  7553. + spin_unlock_local(&lv->lock);
  7554. +}
  7555. +
  7556. +#define local_unlock(lvar) \
  7557. + do { \
  7558. + __local_unlock(this_cpu_ptr(&lvar)); \
  7559. + put_local_var(lvar); \
  7560. + } while (0)
  7561. +
  7562. +#define local_unlock_on(lvar, cpu) \
  7563. + do { __local_unlock(&per_cpu(lvar, cpu)); } while (0)
  7564. +
  7565. +static inline void __local_lock_irq(struct local_irq_lock *lv)
  7566. +{
  7567. + spin_lock_irqsave(&lv->lock, lv->flags);
  7568. + LL_WARN(lv->owner);
  7569. + LL_WARN(lv->nestcnt);
  7570. + lv->owner = current;
  7571. + lv->nestcnt = 1;
  7572. +}
  7573. +
  7574. +#define local_lock_irq(lvar) \
  7575. + do { __local_lock_irq(&get_local_var(lvar)); } while (0)
  7576. +
  7577. +#define local_lock_irq_on(lvar, cpu) \
  7578. + do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0)
  7579. +
  7580. +static inline void __local_unlock_irq(struct local_irq_lock *lv)
  7581. +{
  7582. + LL_WARN(!lv->nestcnt);
  7583. + LL_WARN(lv->owner != current);
  7584. + lv->owner = NULL;
  7585. + lv->nestcnt = 0;
  7586. + spin_unlock_irq(&lv->lock);
  7587. +}
  7588. +
  7589. +#define local_unlock_irq(lvar) \
  7590. + do { \
  7591. + __local_unlock_irq(this_cpu_ptr(&lvar)); \
  7592. + put_local_var(lvar); \
  7593. + } while (0)
  7594. +
  7595. +#define local_unlock_irq_on(lvar, cpu) \
  7596. + do { \
  7597. + __local_unlock_irq(&per_cpu(lvar, cpu)); \
  7598. + } while (0)
  7599. +
  7600. +static inline int __local_lock_irqsave(struct local_irq_lock *lv)
  7601. +{
  7602. + if (lv->owner != current) {
  7603. + __local_lock_irq(lv);
  7604. + return 0;
  7605. + } else {
  7606. + lv->nestcnt++;
  7607. + return 1;
  7608. + }
  7609. +}
  7610. +
  7611. +#define local_lock_irqsave(lvar, _flags) \
  7612. + do { \
  7613. + if (__local_lock_irqsave(&get_local_var(lvar))) \
  7614. + put_local_var(lvar); \
  7615. + _flags = __this_cpu_read(lvar.flags); \
  7616. + } while (0)
  7617. +
  7618. +#define local_lock_irqsave_on(lvar, _flags, cpu) \
  7619. + do { \
  7620. + __local_lock_irqsave(&per_cpu(lvar, cpu)); \
  7621. + _flags = per_cpu(lvar, cpu).flags; \
  7622. + } while (0)
  7623. +
  7624. +static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
  7625. + unsigned long flags)
  7626. +{
  7627. + LL_WARN(!lv->nestcnt);
  7628. + LL_WARN(lv->owner != current);
  7629. + if (--lv->nestcnt)
  7630. + return 0;
  7631. +
  7632. + lv->owner = NULL;
  7633. + spin_unlock_irqrestore(&lv->lock, lv->flags);
  7634. + return 1;
  7635. +}
  7636. +
  7637. +#define local_unlock_irqrestore(lvar, flags) \
  7638. + do { \
  7639. + if (__local_unlock_irqrestore(this_cpu_ptr(&lvar), flags)) \
  7640. + put_local_var(lvar); \
  7641. + } while (0)
  7642. +
  7643. +#define local_unlock_irqrestore_on(lvar, flags, cpu) \
  7644. + do { \
  7645. + __local_unlock_irqrestore(&per_cpu(lvar, cpu), flags); \
  7646. + } while (0)
  7647. +
  7648. +#define local_spin_trylock_irq(lvar, lock) \
  7649. + ({ \
  7650. + int __locked; \
  7651. + local_lock_irq(lvar); \
  7652. + __locked = spin_trylock(lock); \
  7653. + if (!__locked) \
  7654. + local_unlock_irq(lvar); \
  7655. + __locked; \
  7656. + })
  7657. +
  7658. +#define local_spin_lock_irq(lvar, lock) \
  7659. + do { \
  7660. + local_lock_irq(lvar); \
  7661. + spin_lock(lock); \
  7662. + } while (0)
  7663. +
  7664. +#define local_spin_unlock_irq(lvar, lock) \
  7665. + do { \
  7666. + spin_unlock(lock); \
  7667. + local_unlock_irq(lvar); \
  7668. + } while (0)
  7669. +
  7670. +#define local_spin_lock_irqsave(lvar, lock, flags) \
  7671. + do { \
  7672. + local_lock_irqsave(lvar, flags); \
  7673. + spin_lock(lock); \
  7674. + } while (0)
  7675. +
  7676. +#define local_spin_unlock_irqrestore(lvar, lock, flags) \
  7677. + do { \
  7678. + spin_unlock(lock); \
  7679. + local_unlock_irqrestore(lvar, flags); \
  7680. + } while (0)
  7681. +
  7682. +#define get_locked_var(lvar, var) \
  7683. + (*({ \
  7684. + local_lock(lvar); \
  7685. + this_cpu_ptr(&var); \
  7686. + }))
  7687. +
  7688. +#define put_locked_var(lvar, var) local_unlock(lvar);
  7689. +
  7690. +#define local_lock_cpu(lvar) \
  7691. + ({ \
  7692. + local_lock(lvar); \
  7693. + smp_processor_id(); \
  7694. + })
  7695. +
  7696. +#define local_unlock_cpu(lvar) local_unlock(lvar)
  7697. +
  7698. +#else /* PREEMPT_RT_BASE */
  7699. +
  7700. +#define DEFINE_LOCAL_IRQ_LOCK(lvar) __typeof__(const int) lvar
  7701. +#define DECLARE_LOCAL_IRQ_LOCK(lvar) extern __typeof__(const int) lvar
  7702. +
  7703. +static inline void local_irq_lock_init(int lvar) { }
  7704. +
  7705. +#define local_lock(lvar) preempt_disable()
  7706. +#define local_unlock(lvar) preempt_enable()
  7707. +#define local_lock_irq(lvar) local_irq_disable()
  7708. +#define local_lock_irq_on(lvar, cpu) local_irq_disable()
  7709. +#define local_unlock_irq(lvar) local_irq_enable()
  7710. +#define local_unlock_irq_on(lvar, cpu) local_irq_enable()
  7711. +#define local_lock_irqsave(lvar, flags) local_irq_save(flags)
  7712. +#define local_unlock_irqrestore(lvar, flags) local_irq_restore(flags)
  7713. +
  7714. +#define local_spin_trylock_irq(lvar, lock) spin_trylock_irq(lock)
  7715. +#define local_spin_lock_irq(lvar, lock) spin_lock_irq(lock)
  7716. +#define local_spin_unlock_irq(lvar, lock) spin_unlock_irq(lock)
  7717. +#define local_spin_lock_irqsave(lvar, lock, flags) \
  7718. + spin_lock_irqsave(lock, flags)
  7719. +#define local_spin_unlock_irqrestore(lvar, lock, flags) \
  7720. + spin_unlock_irqrestore(lock, flags)
  7721. +
  7722. +#define get_locked_var(lvar, var) get_cpu_var(var)
  7723. +#define put_locked_var(lvar, var) put_cpu_var(var)
  7724. +
  7725. +#define local_lock_cpu(lvar) get_cpu()
  7726. +#define local_unlock_cpu(lvar) put_cpu()
  7727. +
  7728. +#endif
  7729. +
  7730. +#endif
  7731. diff -Nur linux-4.6.6.orig/include/linux/mm_types.h linux-4.6.6/include/linux/mm_types.h
  7732. --- linux-4.6.6.orig/include/linux/mm_types.h 2016-08-10 12:54:57.000000000 +0200
  7733. +++ linux-4.6.6/include/linux/mm_types.h 2016-08-22 21:27:47.321887212 +0200
  7734. @@ -11,6 +11,7 @@
  7735. #include <linux/completion.h>
  7736. #include <linux/cpumask.h>
  7737. #include <linux/uprobes.h>
  7738. +#include <linux/rcupdate.h>
  7739. #include <linux/page-flags-layout.h>
  7740. #include <asm/page.h>
  7741. #include <asm/mmu.h>
  7742. @@ -502,6 +503,9 @@
  7743. bool tlb_flush_pending;
  7744. #endif
  7745. struct uprobes_state uprobes_state;
  7746. +#ifdef CONFIG_PREEMPT_RT_BASE
  7747. + struct rcu_head delayed_drop;
  7748. +#endif
  7749. #ifdef CONFIG_X86_INTEL_MPX
  7750. /* address of the bounds directory */
  7751. void __user *bd_addr;
  7752. diff -Nur linux-4.6.6.orig/include/linux/mutex.h linux-4.6.6/include/linux/mutex.h
  7753. --- linux-4.6.6.orig/include/linux/mutex.h 2016-08-10 12:54:57.000000000 +0200
  7754. +++ linux-4.6.6/include/linux/mutex.h 2016-08-22 21:27:47.321887212 +0200
  7755. @@ -19,6 +19,17 @@
  7756. #include <asm/processor.h>
  7757. #include <linux/osq_lock.h>
  7758. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7759. +# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
  7760. + , .dep_map = { .name = #lockname }
  7761. +#else
  7762. +# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
  7763. +#endif
  7764. +
  7765. +#ifdef CONFIG_PREEMPT_RT_FULL
  7766. +# include <linux/mutex_rt.h>
  7767. +#else
  7768. +
  7769. /*
  7770. * Simple, straightforward mutexes with strict semantics:
  7771. *
  7772. @@ -99,13 +110,6 @@
  7773. static inline void mutex_destroy(struct mutex *lock) {}
  7774. #endif
  7775. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7776. -# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
  7777. - , .dep_map = { .name = #lockname }
  7778. -#else
  7779. -# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
  7780. -#endif
  7781. -
  7782. #define __MUTEX_INITIALIZER(lockname) \
  7783. { .count = ATOMIC_INIT(1) \
  7784. , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \
  7785. @@ -173,6 +177,8 @@
  7786. extern int mutex_trylock(struct mutex *lock);
  7787. extern void mutex_unlock(struct mutex *lock);
  7788. +#endif /* !PREEMPT_RT_FULL */
  7789. +
  7790. extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
  7791. #endif /* __LINUX_MUTEX_H */
  7792. diff -Nur linux-4.6.6.orig/include/linux/mutex_rt.h linux-4.6.6/include/linux/mutex_rt.h
  7793. --- linux-4.6.6.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100
  7794. +++ linux-4.6.6/include/linux/mutex_rt.h 2016-08-22 21:27:47.321887212 +0200
  7795. @@ -0,0 +1,84 @@
  7796. +#ifndef __LINUX_MUTEX_RT_H
  7797. +#define __LINUX_MUTEX_RT_H
  7798. +
  7799. +#ifndef __LINUX_MUTEX_H
  7800. +#error "Please include mutex.h"
  7801. +#endif
  7802. +
  7803. +#include <linux/rtmutex.h>
  7804. +
  7805. +/* FIXME: Just for __lockfunc */
  7806. +#include <linux/spinlock.h>
  7807. +
  7808. +struct mutex {
  7809. + struct rt_mutex lock;
  7810. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7811. + struct lockdep_map dep_map;
  7812. +#endif
  7813. +};
  7814. +
  7815. +#define __MUTEX_INITIALIZER(mutexname) \
  7816. + { \
  7817. + .lock = __RT_MUTEX_INITIALIZER(mutexname.lock) \
  7818. + __DEP_MAP_MUTEX_INITIALIZER(mutexname) \
  7819. + }
  7820. +
  7821. +#define DEFINE_MUTEX(mutexname) \
  7822. + struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
  7823. +
  7824. +extern void __mutex_do_init(struct mutex *lock, const char *name, struct lock_class_key *key);
  7825. +extern void __lockfunc _mutex_lock(struct mutex *lock);
  7826. +extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock);
  7827. +extern int __lockfunc _mutex_lock_killable(struct mutex *lock);
  7828. +extern void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass);
  7829. +extern void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
  7830. +extern int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass);
  7831. +extern int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass);
  7832. +extern int __lockfunc _mutex_trylock(struct mutex *lock);
  7833. +extern void __lockfunc _mutex_unlock(struct mutex *lock);
  7834. +
  7835. +#define mutex_is_locked(l) rt_mutex_is_locked(&(l)->lock)
  7836. +#define mutex_lock(l) _mutex_lock(l)
  7837. +#define mutex_lock_interruptible(l) _mutex_lock_interruptible(l)
  7838. +#define mutex_lock_killable(l) _mutex_lock_killable(l)
  7839. +#define mutex_trylock(l) _mutex_trylock(l)
  7840. +#define mutex_unlock(l) _mutex_unlock(l)
  7841. +#define mutex_destroy(l) rt_mutex_destroy(&(l)->lock)
  7842. +
  7843. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7844. +# define mutex_lock_nested(l, s) _mutex_lock_nested(l, s)
  7845. +# define mutex_lock_interruptible_nested(l, s) \
  7846. + _mutex_lock_interruptible_nested(l, s)
  7847. +# define mutex_lock_killable_nested(l, s) \
  7848. + _mutex_lock_killable_nested(l, s)
  7849. +
  7850. +# define mutex_lock_nest_lock(lock, nest_lock) \
  7851. +do { \
  7852. + typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
  7853. + _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \
  7854. +} while (0)
  7855. +
  7856. +#else
  7857. +# define mutex_lock_nested(l, s) _mutex_lock(l)
  7858. +# define mutex_lock_interruptible_nested(l, s) \
  7859. + _mutex_lock_interruptible(l)
  7860. +# define mutex_lock_killable_nested(l, s) \
  7861. + _mutex_lock_killable(l)
  7862. +# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock)
  7863. +#endif
  7864. +
  7865. +# define mutex_init(mutex) \
  7866. +do { \
  7867. + static struct lock_class_key __key; \
  7868. + \
  7869. + rt_mutex_init(&(mutex)->lock); \
  7870. + __mutex_do_init((mutex), #mutex, &__key); \
  7871. +} while (0)
  7872. +
  7873. +# define __mutex_init(mutex, name, key) \
  7874. +do { \
  7875. + rt_mutex_init(&(mutex)->lock); \
  7876. + __mutex_do_init((mutex), name, key); \
  7877. +} while (0)
  7878. +
  7879. +#endif
  7880. diff -Nur linux-4.6.6.orig/include/linux/netdevice.h linux-4.6.6/include/linux/netdevice.h
  7881. --- linux-4.6.6.orig/include/linux/netdevice.h 2016-08-10 12:54:57.000000000 +0200
  7882. +++ linux-4.6.6/include/linux/netdevice.h 2016-08-22 21:27:47.325887472 +0200
  7883. @@ -2396,11 +2396,20 @@
  7884. void synchronize_net(void);
  7885. int init_dummy_netdev(struct net_device *dev);
  7886. +#ifdef CONFIG_PREEMPT_RT_FULL
  7887. +static inline int dev_recursion_level(void)
  7888. +{
  7889. + return current->xmit_recursion;
  7890. +}
  7891. +
  7892. +#else
  7893. +
  7894. DECLARE_PER_CPU(int, xmit_recursion);
  7895. static inline int dev_recursion_level(void)
  7896. {
  7897. return this_cpu_read(xmit_recursion);
  7898. }
  7899. +#endif
  7900. struct net_device *dev_get_by_index(struct net *net, int ifindex);
  7901. struct net_device *__dev_get_by_index(struct net *net, int ifindex);
  7902. @@ -2776,6 +2785,7 @@
  7903. unsigned int dropped;
  7904. struct sk_buff_head input_pkt_queue;
  7905. struct napi_struct backlog;
  7906. + struct sk_buff_head tofree_queue;
  7907. };
  7908. diff -Nur linux-4.6.6.orig/include/linux/netfilter/x_tables.h linux-4.6.6/include/linux/netfilter/x_tables.h
  7909. --- linux-4.6.6.orig/include/linux/netfilter/x_tables.h 2016-08-10 12:54:57.000000000 +0200
  7910. +++ linux-4.6.6/include/linux/netfilter/x_tables.h 2016-08-22 21:27:47.325887472 +0200
  7911. @@ -4,6 +4,7 @@
  7912. #include <linux/netdevice.h>
  7913. #include <linux/static_key.h>
  7914. +#include <linux/locallock.h>
  7915. #include <uapi/linux/netfilter/x_tables.h>
  7916. /**
  7917. @@ -292,6 +293,8 @@
  7918. */
  7919. DECLARE_PER_CPU(seqcount_t, xt_recseq);
  7920. +DECLARE_LOCAL_IRQ_LOCK(xt_write_lock);
  7921. +
  7922. /* xt_tee_enabled - true if x_tables needs to handle reentrancy
  7923. *
  7924. * Enabled if current ip(6)tables ruleset has at least one -j TEE rule.
  7925. @@ -312,6 +315,9 @@
  7926. {
  7927. unsigned int addend;
  7928. + /* RT protection */
  7929. + local_lock(xt_write_lock);
  7930. +
  7931. /*
  7932. * Low order bit of sequence is set if we already
  7933. * called xt_write_recseq_begin().
  7934. @@ -342,6 +348,7 @@
  7935. /* this is kind of a write_seqcount_end(), but addend is 0 or 1 */
  7936. smp_wmb();
  7937. __this_cpu_add(xt_recseq.sequence, addend);
  7938. + local_unlock(xt_write_lock);
  7939. }
  7940. /*
  7941. diff -Nur linux-4.6.6.orig/include/linux/notifier.h linux-4.6.6/include/linux/notifier.h
  7942. --- linux-4.6.6.orig/include/linux/notifier.h 2016-08-10 12:54:57.000000000 +0200
  7943. +++ linux-4.6.6/include/linux/notifier.h 2016-08-22 21:27:47.325887472 +0200
  7944. @@ -6,7 +6,7 @@
  7945. *
  7946. * Alan Cox <Alan.Cox@linux.org>
  7947. */
  7948. -
  7949. +
  7950. #ifndef _LINUX_NOTIFIER_H
  7951. #define _LINUX_NOTIFIER_H
  7952. #include <linux/errno.h>
  7953. @@ -42,9 +42,7 @@
  7954. * in srcu_notifier_call_chain(): no cache bounces and no memory barriers.
  7955. * As compensation, srcu_notifier_chain_unregister() is rather expensive.
  7956. * SRCU notifier chains should be used when the chain will be called very
  7957. - * often but notifier_blocks will seldom be removed. Also, SRCU notifier
  7958. - * chains are slightly more difficult to use because they require special
  7959. - * runtime initialization.
  7960. + * often but notifier_blocks will seldom be removed.
  7961. */
  7962. struct notifier_block;
  7963. @@ -90,7 +88,7 @@
  7964. (name)->head = NULL; \
  7965. } while (0)
  7966. -/* srcu_notifier_heads must be initialized and cleaned up dynamically */
  7967. +/* srcu_notifier_heads must be cleaned up dynamically */
  7968. extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
  7969. #define srcu_cleanup_notifier_head(name) \
  7970. cleanup_srcu_struct(&(name)->srcu);
  7971. @@ -103,7 +101,13 @@
  7972. .head = NULL }
  7973. #define RAW_NOTIFIER_INIT(name) { \
  7974. .head = NULL }
  7975. -/* srcu_notifier_heads cannot be initialized statically */
  7976. +
  7977. +#define SRCU_NOTIFIER_INIT(name, pcpu) \
  7978. + { \
  7979. + .mutex = __MUTEX_INITIALIZER(name.mutex), \
  7980. + .head = NULL, \
  7981. + .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \
  7982. + }
  7983. #define ATOMIC_NOTIFIER_HEAD(name) \
  7984. struct atomic_notifier_head name = \
  7985. @@ -115,6 +119,18 @@
  7986. struct raw_notifier_head name = \
  7987. RAW_NOTIFIER_INIT(name)
  7988. +#define _SRCU_NOTIFIER_HEAD(name, mod) \
  7989. + static DEFINE_PER_CPU(struct srcu_struct_array, \
  7990. + name##_head_srcu_array); \
  7991. + mod struct srcu_notifier_head name = \
  7992. + SRCU_NOTIFIER_INIT(name, name##_head_srcu_array)
  7993. +
  7994. +#define SRCU_NOTIFIER_HEAD(name) \
  7995. + _SRCU_NOTIFIER_HEAD(name, )
  7996. +
  7997. +#define SRCU_NOTIFIER_HEAD_STATIC(name) \
  7998. + _SRCU_NOTIFIER_HEAD(name, static)
  7999. +
  8000. #ifdef __KERNEL__
  8001. extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
  8002. @@ -184,12 +200,12 @@
  8003. /*
  8004. * Declared notifiers so far. I can imagine quite a few more chains
  8005. - * over time (eg laptop power reset chains, reboot chain (to clean
  8006. + * over time (eg laptop power reset chains, reboot chain (to clean
  8007. * device units up), device [un]mount chain, module load/unload chain,
  8008. - * low memory chain, screenblank chain (for plug in modular screenblankers)
  8009. + * low memory chain, screenblank chain (for plug in modular screenblankers)
  8010. * VC switch chains (for loadable kernel svgalib VC switch helpers) etc...
  8011. */
  8012. -
  8013. +
  8014. /* CPU notfiers are defined in include/linux/cpu.h. */
  8015. /* netdevice notifiers are defined in include/linux/netdevice.h */
  8016. diff -Nur linux-4.6.6.orig/include/linux/percpu.h linux-4.6.6/include/linux/percpu.h
  8017. --- linux-4.6.6.orig/include/linux/percpu.h 2016-08-10 12:54:57.000000000 +0200
  8018. +++ linux-4.6.6/include/linux/percpu.h 2016-08-22 21:27:47.325887472 +0200
  8019. @@ -18,6 +18,35 @@
  8020. #define PERCPU_MODULE_RESERVE 0
  8021. #endif
  8022. +#ifdef CONFIG_PREEMPT_RT_FULL
  8023. +
  8024. +#define get_local_var(var) (*({ \
  8025. + migrate_disable(); \
  8026. + this_cpu_ptr(&var); }))
  8027. +
  8028. +#define put_local_var(var) do { \
  8029. + (void)&(var); \
  8030. + migrate_enable(); \
  8031. +} while (0)
  8032. +
  8033. +# define get_local_ptr(var) ({ \
  8034. + migrate_disable(); \
  8035. + this_cpu_ptr(var); })
  8036. +
  8037. +# define put_local_ptr(var) do { \
  8038. + (void)(var); \
  8039. + migrate_enable(); \
  8040. +} while (0)
  8041. +
  8042. +#else
  8043. +
  8044. +#define get_local_var(var) get_cpu_var(var)
  8045. +#define put_local_var(var) put_cpu_var(var)
  8046. +#define get_local_ptr(var) get_cpu_ptr(var)
  8047. +#define put_local_ptr(var) put_cpu_ptr(var)
  8048. +
  8049. +#endif
  8050. +
  8051. /* minimum unit size, also is the maximum supported allocation size */
  8052. #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
  8053. diff -Nur linux-4.6.6.orig/include/linux/pid.h linux-4.6.6/include/linux/pid.h
  8054. --- linux-4.6.6.orig/include/linux/pid.h 2016-08-10 12:54:57.000000000 +0200
  8055. +++ linux-4.6.6/include/linux/pid.h 2016-08-22 21:27:47.325887472 +0200
  8056. @@ -2,6 +2,7 @@
  8057. #define _LINUX_PID_H
  8058. #include <linux/rcupdate.h>
  8059. +#include <linux/atomic.h>
  8060. enum pid_type
  8061. {
  8062. diff -Nur linux-4.6.6.orig/include/linux/preempt.h linux-4.6.6/include/linux/preempt.h
  8063. --- linux-4.6.6.orig/include/linux/preempt.h 2016-08-10 12:54:57.000000000 +0200
  8064. +++ linux-4.6.6/include/linux/preempt.h 2016-08-22 21:27:47.325887472 +0200
  8065. @@ -50,7 +50,11 @@
  8066. #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
  8067. #define NMI_OFFSET (1UL << NMI_SHIFT)
  8068. -#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
  8069. +#ifndef CONFIG_PREEMPT_RT_FULL
  8070. +# define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
  8071. +#else
  8072. +# define SOFTIRQ_DISABLE_OFFSET (0)
  8073. +#endif
  8074. /* We use the MSB mostly because its available */
  8075. #define PREEMPT_NEED_RESCHED 0x80000000
  8076. @@ -59,9 +63,15 @@
  8077. #include <asm/preempt.h>
  8078. #define hardirq_count() (preempt_count() & HARDIRQ_MASK)
  8079. -#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
  8080. #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
  8081. | NMI_MASK))
  8082. +#ifndef CONFIG_PREEMPT_RT_FULL
  8083. +# define softirq_count() (preempt_count() & SOFTIRQ_MASK)
  8084. +# define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
  8085. +#else
  8086. +# define softirq_count() (0UL)
  8087. +extern int in_serving_softirq(void);
  8088. +#endif
  8089. /*
  8090. * Are we doing bottom half or hardware interrupt processing?
  8091. @@ -72,7 +82,6 @@
  8092. #define in_irq() (hardirq_count())
  8093. #define in_softirq() (softirq_count())
  8094. #define in_interrupt() (irq_count())
  8095. -#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
  8096. /*
  8097. * Are we in NMI context?
  8098. @@ -91,7 +100,11 @@
  8099. /*
  8100. * The preempt_count offset after spin_lock()
  8101. */
  8102. +#if !defined(CONFIG_PREEMPT_RT_FULL)
  8103. #define PREEMPT_LOCK_OFFSET PREEMPT_DISABLE_OFFSET
  8104. +#else
  8105. +#define PREEMPT_LOCK_OFFSET 0
  8106. +#endif
  8107. /*
  8108. * The preempt_count offset needed for things like:
  8109. @@ -140,6 +153,20 @@
  8110. #define preempt_count_inc() preempt_count_add(1)
  8111. #define preempt_count_dec() preempt_count_sub(1)
  8112. +#ifdef CONFIG_PREEMPT_LAZY
  8113. +#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0)
  8114. +#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0)
  8115. +#define inc_preempt_lazy_count() add_preempt_lazy_count(1)
  8116. +#define dec_preempt_lazy_count() sub_preempt_lazy_count(1)
  8117. +#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count)
  8118. +#else
  8119. +#define add_preempt_lazy_count(val) do { } while (0)
  8120. +#define sub_preempt_lazy_count(val) do { } while (0)
  8121. +#define inc_preempt_lazy_count() do { } while (0)
  8122. +#define dec_preempt_lazy_count() do { } while (0)
  8123. +#define preempt_lazy_count() (0)
  8124. +#endif
  8125. +
  8126. #ifdef CONFIG_PREEMPT_COUNT
  8127. #define preempt_disable() \
  8128. @@ -148,13 +175,25 @@
  8129. barrier(); \
  8130. } while (0)
  8131. +#define preempt_lazy_disable() \
  8132. +do { \
  8133. + inc_preempt_lazy_count(); \
  8134. + barrier(); \
  8135. +} while (0)
  8136. +
  8137. #define sched_preempt_enable_no_resched() \
  8138. do { \
  8139. barrier(); \
  8140. preempt_count_dec(); \
  8141. } while (0)
  8142. -#define preempt_enable_no_resched() sched_preempt_enable_no_resched()
  8143. +#ifdef CONFIG_PREEMPT_RT_BASE
  8144. +# define preempt_enable_no_resched() sched_preempt_enable_no_resched()
  8145. +# define preempt_check_resched_rt() preempt_check_resched()
  8146. +#else
  8147. +# define preempt_enable_no_resched() preempt_enable()
  8148. +# define preempt_check_resched_rt() barrier();
  8149. +#endif
  8150. #define preemptible() (preempt_count() == 0 && !irqs_disabled())
  8151. @@ -179,6 +218,13 @@
  8152. __preempt_schedule(); \
  8153. } while (0)
  8154. +#define preempt_lazy_enable() \
  8155. +do { \
  8156. + dec_preempt_lazy_count(); \
  8157. + barrier(); \
  8158. + preempt_check_resched(); \
  8159. +} while (0)
  8160. +
  8161. #else /* !CONFIG_PREEMPT */
  8162. #define preempt_enable() \
  8163. do { \
  8164. @@ -224,6 +270,7 @@
  8165. #define preempt_disable_notrace() barrier()
  8166. #define preempt_enable_no_resched_notrace() barrier()
  8167. #define preempt_enable_notrace() barrier()
  8168. +#define preempt_check_resched_rt() barrier()
  8169. #define preemptible() 0
  8170. #endif /* CONFIG_PREEMPT_COUNT */
  8171. @@ -244,10 +291,31 @@
  8172. } while (0)
  8173. #define preempt_fold_need_resched() \
  8174. do { \
  8175. - if (tif_need_resched()) \
  8176. + if (tif_need_resched_now()) \
  8177. set_preempt_need_resched(); \
  8178. } while (0)
  8179. +#ifdef CONFIG_PREEMPT_RT_FULL
  8180. +# define preempt_disable_rt() preempt_disable()
  8181. +# define preempt_enable_rt() preempt_enable()
  8182. +# define preempt_disable_nort() barrier()
  8183. +# define preempt_enable_nort() barrier()
  8184. +# ifdef CONFIG_SMP
  8185. + extern void migrate_disable(void);
  8186. + extern void migrate_enable(void);
  8187. +# else /* CONFIG_SMP */
  8188. +# define migrate_disable() barrier()
  8189. +# define migrate_enable() barrier()
  8190. +# endif /* CONFIG_SMP */
  8191. +#else
  8192. +# define preempt_disable_rt() barrier()
  8193. +# define preempt_enable_rt() barrier()
  8194. +# define preempt_disable_nort() preempt_disable()
  8195. +# define preempt_enable_nort() preempt_enable()
  8196. +# define migrate_disable() preempt_disable()
  8197. +# define migrate_enable() preempt_enable()
  8198. +#endif
  8199. +
  8200. #ifdef CONFIG_PREEMPT_NOTIFIERS
  8201. struct preempt_notifier;
  8202. diff -Nur linux-4.6.6.orig/include/linux/printk.h linux-4.6.6/include/linux/printk.h
  8203. --- linux-4.6.6.orig/include/linux/printk.h 2016-08-10 12:54:57.000000000 +0200
  8204. +++ linux-4.6.6/include/linux/printk.h 2016-08-22 21:27:47.329887725 +0200
  8205. @@ -117,9 +117,11 @@
  8206. #ifdef CONFIG_EARLY_PRINTK
  8207. extern asmlinkage __printf(1, 2)
  8208. void early_printk(const char *fmt, ...);
  8209. +extern void printk_kill(void);
  8210. #else
  8211. static inline __printf(1, 2) __cold
  8212. void early_printk(const char *s, ...) { }
  8213. +static inline void printk_kill(void) { }
  8214. #endif
  8215. typedef __printf(1, 0) int (*printk_func_t)(const char *fmt, va_list args);
  8216. diff -Nur linux-4.6.6.orig/include/linux/radix-tree.h linux-4.6.6/include/linux/radix-tree.h
  8217. --- linux-4.6.6.orig/include/linux/radix-tree.h 2016-08-10 12:54:57.000000000 +0200
  8218. +++ linux-4.6.6/include/linux/radix-tree.h 2016-08-22 21:27:47.329887725 +0200
  8219. @@ -294,8 +294,13 @@
  8220. unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
  8221. void ***results, unsigned long *indices,
  8222. unsigned long first_index, unsigned int max_items);
  8223. +#ifndef CONFIG_PREEMPT_RT_FULL
  8224. int radix_tree_preload(gfp_t gfp_mask);
  8225. int radix_tree_maybe_preload(gfp_t gfp_mask);
  8226. +#else
  8227. +static inline int radix_tree_preload(gfp_t gm) { return 0; }
  8228. +static inline int radix_tree_maybe_preload(gfp_t gfp_mask) { return 0; }
  8229. +#endif
  8230. void radix_tree_init(void);
  8231. void *radix_tree_tag_set(struct radix_tree_root *root,
  8232. unsigned long index, unsigned int tag);
  8233. @@ -320,7 +325,7 @@
  8234. static inline void radix_tree_preload_end(void)
  8235. {
  8236. - preempt_enable();
  8237. + preempt_enable_nort();
  8238. }
  8239. /**
  8240. diff -Nur linux-4.6.6.orig/include/linux/random.h linux-4.6.6/include/linux/random.h
  8241. --- linux-4.6.6.orig/include/linux/random.h 2016-08-10 12:54:57.000000000 +0200
  8242. +++ linux-4.6.6/include/linux/random.h 2016-08-22 21:27:47.329887725 +0200
  8243. @@ -20,7 +20,7 @@
  8244. extern void add_device_randomness(const void *, unsigned int);
  8245. extern void add_input_randomness(unsigned int type, unsigned int code,
  8246. unsigned int value);
  8247. -extern void add_interrupt_randomness(int irq, int irq_flags);
  8248. +extern void add_interrupt_randomness(int irq, int irq_flags, __u64 ip);
  8249. extern void get_random_bytes(void *buf, int nbytes);
  8250. extern int add_random_ready_callback(struct random_ready_callback *rdy);
  8251. diff -Nur linux-4.6.6.orig/include/linux/rbtree.h linux-4.6.6/include/linux/rbtree.h
  8252. --- linux-4.6.6.orig/include/linux/rbtree.h 2016-08-10 12:54:57.000000000 +0200
  8253. +++ linux-4.6.6/include/linux/rbtree.h 2016-08-22 21:27:47.329887725 +0200
  8254. @@ -31,7 +31,6 @@
  8255. #include <linux/kernel.h>
  8256. #include <linux/stddef.h>
  8257. -#include <linux/rcupdate.h>
  8258. struct rb_node {
  8259. unsigned long __rb_parent_color;
  8260. @@ -86,14 +85,8 @@
  8261. *rb_link = node;
  8262. }
  8263. -static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,
  8264. - struct rb_node **rb_link)
  8265. -{
  8266. - node->__rb_parent_color = (unsigned long)parent;
  8267. - node->rb_left = node->rb_right = NULL;
  8268. -
  8269. - rcu_assign_pointer(*rb_link, node);
  8270. -}
  8271. +void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,
  8272. + struct rb_node **rb_link);
  8273. #define rb_entry_safe(ptr, type, member) \
  8274. ({ typeof(ptr) ____ptr = (ptr); \
  8275. diff -Nur linux-4.6.6.orig/include/linux/rcupdate.h linux-4.6.6/include/linux/rcupdate.h
  8276. --- linux-4.6.6.orig/include/linux/rcupdate.h 2016-08-10 12:54:57.000000000 +0200
  8277. +++ linux-4.6.6/include/linux/rcupdate.h 2016-08-22 21:27:47.329887725 +0200
  8278. @@ -177,6 +177,9 @@
  8279. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  8280. +#ifdef CONFIG_PREEMPT_RT_FULL
  8281. +#define call_rcu_bh call_rcu
  8282. +#else
  8283. /**
  8284. * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
  8285. * @head: structure to be used for queueing the RCU updates.
  8286. @@ -200,6 +203,7 @@
  8287. */
  8288. void call_rcu_bh(struct rcu_head *head,
  8289. rcu_callback_t func);
  8290. +#endif
  8291. /**
  8292. * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
  8293. @@ -300,6 +304,11 @@
  8294. * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
  8295. */
  8296. #define rcu_preempt_depth() (current->rcu_read_lock_nesting)
  8297. +#ifndef CONFIG_PREEMPT_RT_FULL
  8298. +#define sched_rcu_preempt_depth() rcu_preempt_depth()
  8299. +#else
  8300. +static inline int sched_rcu_preempt_depth(void) { return 0; }
  8301. +#endif
  8302. #else /* #ifdef CONFIG_PREEMPT_RCU */
  8303. @@ -325,6 +334,8 @@
  8304. return 0;
  8305. }
  8306. +#define sched_rcu_preempt_depth() rcu_preempt_depth()
  8307. +
  8308. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  8309. /* Internal to kernel */
  8310. @@ -498,7 +509,14 @@
  8311. int debug_lockdep_rcu_enabled(void);
  8312. int rcu_read_lock_held(void);
  8313. +#ifdef CONFIG_PREEMPT_RT_FULL
  8314. +static inline int rcu_read_lock_bh_held(void)
  8315. +{
  8316. + return rcu_read_lock_held();
  8317. +}
  8318. +#else
  8319. int rcu_read_lock_bh_held(void);
  8320. +#endif
  8321. /**
  8322. * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
  8323. @@ -946,10 +964,14 @@
  8324. static inline void rcu_read_lock_bh(void)
  8325. {
  8326. local_bh_disable();
  8327. +#ifdef CONFIG_PREEMPT_RT_FULL
  8328. + rcu_read_lock();
  8329. +#else
  8330. __acquire(RCU_BH);
  8331. rcu_lock_acquire(&rcu_bh_lock_map);
  8332. RCU_LOCKDEP_WARN(!rcu_is_watching(),
  8333. "rcu_read_lock_bh() used illegally while idle");
  8334. +#endif
  8335. }
  8336. /*
  8337. @@ -959,10 +981,14 @@
  8338. */
  8339. static inline void rcu_read_unlock_bh(void)
  8340. {
  8341. +#ifdef CONFIG_PREEMPT_RT_FULL
  8342. + rcu_read_unlock();
  8343. +#else
  8344. RCU_LOCKDEP_WARN(!rcu_is_watching(),
  8345. "rcu_read_unlock_bh() used illegally while idle");
  8346. rcu_lock_release(&rcu_bh_lock_map);
  8347. __release(RCU_BH);
  8348. +#endif
  8349. local_bh_enable();
  8350. }
  8351. diff -Nur linux-4.6.6.orig/include/linux/rcutree.h linux-4.6.6/include/linux/rcutree.h
  8352. --- linux-4.6.6.orig/include/linux/rcutree.h 2016-08-10 12:54:57.000000000 +0200
  8353. +++ linux-4.6.6/include/linux/rcutree.h 2016-08-22 21:27:47.329887725 +0200
  8354. @@ -44,7 +44,11 @@
  8355. rcu_note_context_switch();
  8356. }
  8357. +#ifdef CONFIG_PREEMPT_RT_FULL
  8358. +# define synchronize_rcu_bh synchronize_rcu
  8359. +#else
  8360. void synchronize_rcu_bh(void);
  8361. +#endif
  8362. void synchronize_sched_expedited(void);
  8363. void synchronize_rcu_expedited(void);
  8364. @@ -72,7 +76,11 @@
  8365. }
  8366. void rcu_barrier(void);
  8367. +#ifdef CONFIG_PREEMPT_RT_FULL
  8368. +# define rcu_barrier_bh rcu_barrier
  8369. +#else
  8370. void rcu_barrier_bh(void);
  8371. +#endif
  8372. void rcu_barrier_sched(void);
  8373. unsigned long get_state_synchronize_rcu(void);
  8374. void cond_synchronize_rcu(unsigned long oldstate);
  8375. @@ -85,12 +93,10 @@
  8376. unsigned long rcu_batches_started_bh(void);
  8377. unsigned long rcu_batches_started_sched(void);
  8378. unsigned long rcu_batches_completed(void);
  8379. -unsigned long rcu_batches_completed_bh(void);
  8380. unsigned long rcu_batches_completed_sched(void);
  8381. void show_rcu_gp_kthreads(void);
  8382. void rcu_force_quiescent_state(void);
  8383. -void rcu_bh_force_quiescent_state(void);
  8384. void rcu_sched_force_quiescent_state(void);
  8385. void rcu_idle_enter(void);
  8386. @@ -107,6 +113,14 @@
  8387. bool rcu_is_watching(void);
  8388. +#ifndef CONFIG_PREEMPT_RT_FULL
  8389. +void rcu_bh_force_quiescent_state(void);
  8390. +unsigned long rcu_batches_completed_bh(void);
  8391. +#else
  8392. +# define rcu_bh_force_quiescent_state rcu_force_quiescent_state
  8393. +# define rcu_batches_completed_bh rcu_batches_completed
  8394. +#endif
  8395. +
  8396. void rcu_all_qs(void);
  8397. #endif /* __LINUX_RCUTREE_H */
  8398. diff -Nur linux-4.6.6.orig/include/linux/rtmutex.h linux-4.6.6/include/linux/rtmutex.h
  8399. --- linux-4.6.6.orig/include/linux/rtmutex.h 2016-08-10 12:54:57.000000000 +0200
  8400. +++ linux-4.6.6/include/linux/rtmutex.h 2016-08-22 21:27:47.329887725 +0200
  8401. @@ -13,11 +13,15 @@
  8402. #define __LINUX_RT_MUTEX_H
  8403. #include <linux/linkage.h>
  8404. +#include <linux/spinlock_types_raw.h>
  8405. #include <linux/rbtree.h>
  8406. -#include <linux/spinlock_types.h>
  8407. extern int max_lock_depth; /* for sysctl */
  8408. +#ifdef CONFIG_DEBUG_MUTEXES
  8409. +#include <linux/debug_locks.h>
  8410. +#endif
  8411. +
  8412. /**
  8413. * The rt_mutex structure
  8414. *
  8415. @@ -31,8 +35,8 @@
  8416. struct rb_root waiters;
  8417. struct rb_node *waiters_leftmost;
  8418. struct task_struct *owner;
  8419. -#ifdef CONFIG_DEBUG_RT_MUTEXES
  8420. int save_state;
  8421. +#ifdef CONFIG_DEBUG_RT_MUTEXES
  8422. const char *name, *file;
  8423. int line;
  8424. void *magic;
  8425. @@ -55,22 +59,33 @@
  8426. # define rt_mutex_debug_check_no_locks_held(task) do { } while (0)
  8427. #endif
  8428. +# define rt_mutex_init(mutex) \
  8429. + do { \
  8430. + raw_spin_lock_init(&(mutex)->wait_lock); \
  8431. + __rt_mutex_init(mutex, #mutex); \
  8432. + } while (0)
  8433. +
  8434. #ifdef CONFIG_DEBUG_RT_MUTEXES
  8435. # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \
  8436. , .name = #mutexname, .file = __FILE__, .line = __LINE__
  8437. -# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __func__)
  8438. extern void rt_mutex_debug_task_free(struct task_struct *tsk);
  8439. #else
  8440. # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname)
  8441. -# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL)
  8442. # define rt_mutex_debug_task_free(t) do { } while (0)
  8443. #endif
  8444. -#define __RT_MUTEX_INITIALIZER(mutexname) \
  8445. - { .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \
  8446. +#define __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \
  8447. + .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \
  8448. , .waiters = RB_ROOT \
  8449. , .owner = NULL \
  8450. - __DEBUG_RT_MUTEX_INITIALIZER(mutexname)}
  8451. + __DEBUG_RT_MUTEX_INITIALIZER(mutexname)
  8452. +
  8453. +#define __RT_MUTEX_INITIALIZER(mutexname) \
  8454. + { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) }
  8455. +
  8456. +#define __RT_MUTEX_INITIALIZER_SAVE_STATE(mutexname) \
  8457. + { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \
  8458. + , .save_state = 1 }
  8459. #define DEFINE_RT_MUTEX(mutexname) \
  8460. struct rt_mutex mutexname = __RT_MUTEX_INITIALIZER(mutexname)
  8461. @@ -91,6 +106,7 @@
  8462. extern void rt_mutex_lock(struct rt_mutex *lock);
  8463. extern int rt_mutex_lock_interruptible(struct rt_mutex *lock);
  8464. +extern int rt_mutex_lock_killable(struct rt_mutex *lock);
  8465. extern int rt_mutex_timed_lock(struct rt_mutex *lock,
  8466. struct hrtimer_sleeper *timeout);
  8467. diff -Nur linux-4.6.6.orig/include/linux/rwlock_rt.h linux-4.6.6/include/linux/rwlock_rt.h
  8468. --- linux-4.6.6.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100
  8469. +++ linux-4.6.6/include/linux/rwlock_rt.h 2016-08-22 21:27:47.329887725 +0200
  8470. @@ -0,0 +1,99 @@
  8471. +#ifndef __LINUX_RWLOCK_RT_H
  8472. +#define __LINUX_RWLOCK_RT_H
  8473. +
  8474. +#ifndef __LINUX_SPINLOCK_H
  8475. +#error Do not include directly. Use spinlock.h
  8476. +#endif
  8477. +
  8478. +#define rwlock_init(rwl) \
  8479. +do { \
  8480. + static struct lock_class_key __key; \
  8481. + \
  8482. + rt_mutex_init(&(rwl)->lock); \
  8483. + __rt_rwlock_init(rwl, #rwl, &__key); \
  8484. +} while (0)
  8485. +
  8486. +extern void __lockfunc rt_write_lock(rwlock_t *rwlock);
  8487. +extern void __lockfunc rt_read_lock(rwlock_t *rwlock);
  8488. +extern int __lockfunc rt_write_trylock(rwlock_t *rwlock);
  8489. +extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock, unsigned long *flags);
  8490. +extern int __lockfunc rt_read_trylock(rwlock_t *rwlock);
  8491. +extern void __lockfunc rt_write_unlock(rwlock_t *rwlock);
  8492. +extern void __lockfunc rt_read_unlock(rwlock_t *rwlock);
  8493. +extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock);
  8494. +extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock);
  8495. +extern void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key);
  8496. +
  8497. +#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock))
  8498. +#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock))
  8499. +
  8500. +#define write_trylock_irqsave(lock, flags) \
  8501. + __cond_lock(lock, rt_write_trylock_irqsave(lock, &flags))
  8502. +
  8503. +#define read_lock_irqsave(lock, flags) \
  8504. + do { \
  8505. + typecheck(unsigned long, flags); \
  8506. + flags = rt_read_lock_irqsave(lock); \
  8507. + } while (0)
  8508. +
  8509. +#define write_lock_irqsave(lock, flags) \
  8510. + do { \
  8511. + typecheck(unsigned long, flags); \
  8512. + flags = rt_write_lock_irqsave(lock); \
  8513. + } while (0)
  8514. +
  8515. +#define read_lock(lock) rt_read_lock(lock)
  8516. +
  8517. +#define read_lock_bh(lock) \
  8518. + do { \
  8519. + local_bh_disable(); \
  8520. + rt_read_lock(lock); \
  8521. + } while (0)
  8522. +
  8523. +#define read_lock_irq(lock) read_lock(lock)
  8524. +
  8525. +#define write_lock(lock) rt_write_lock(lock)
  8526. +
  8527. +#define write_lock_bh(lock) \
  8528. + do { \
  8529. + local_bh_disable(); \
  8530. + rt_write_lock(lock); \
  8531. + } while (0)
  8532. +
  8533. +#define write_lock_irq(lock) write_lock(lock)
  8534. +
  8535. +#define read_unlock(lock) rt_read_unlock(lock)
  8536. +
  8537. +#define read_unlock_bh(lock) \
  8538. + do { \
  8539. + rt_read_unlock(lock); \
  8540. + local_bh_enable(); \
  8541. + } while (0)
  8542. +
  8543. +#define read_unlock_irq(lock) read_unlock(lock)
  8544. +
  8545. +#define write_unlock(lock) rt_write_unlock(lock)
  8546. +
  8547. +#define write_unlock_bh(lock) \
  8548. + do { \
  8549. + rt_write_unlock(lock); \
  8550. + local_bh_enable(); \
  8551. + } while (0)
  8552. +
  8553. +#define write_unlock_irq(lock) write_unlock(lock)
  8554. +
  8555. +#define read_unlock_irqrestore(lock, flags) \
  8556. + do { \
  8557. + typecheck(unsigned long, flags); \
  8558. + (void) flags; \
  8559. + rt_read_unlock(lock); \
  8560. + } while (0)
  8561. +
  8562. +#define write_unlock_irqrestore(lock, flags) \
  8563. + do { \
  8564. + typecheck(unsigned long, flags); \
  8565. + (void) flags; \
  8566. + rt_write_unlock(lock); \
  8567. + } while (0)
  8568. +
  8569. +#endif
  8570. diff -Nur linux-4.6.6.orig/include/linux/rwlock_types.h linux-4.6.6/include/linux/rwlock_types.h
  8571. --- linux-4.6.6.orig/include/linux/rwlock_types.h 2016-08-10 12:54:57.000000000 +0200
  8572. +++ linux-4.6.6/include/linux/rwlock_types.h 2016-08-22 21:27:47.329887725 +0200
  8573. @@ -1,6 +1,10 @@
  8574. #ifndef __LINUX_RWLOCK_TYPES_H
  8575. #define __LINUX_RWLOCK_TYPES_H
  8576. +#if !defined(__LINUX_SPINLOCK_TYPES_H)
  8577. +# error "Do not include directly, include spinlock_types.h"
  8578. +#endif
  8579. +
  8580. /*
  8581. * include/linux/rwlock_types.h - generic rwlock type definitions
  8582. * and initializers
  8583. diff -Nur linux-4.6.6.orig/include/linux/rwlock_types_rt.h linux-4.6.6/include/linux/rwlock_types_rt.h
  8584. --- linux-4.6.6.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100
  8585. +++ linux-4.6.6/include/linux/rwlock_types_rt.h 2016-08-22 21:27:47.333887985 +0200
  8586. @@ -0,0 +1,33 @@
  8587. +#ifndef __LINUX_RWLOCK_TYPES_RT_H
  8588. +#define __LINUX_RWLOCK_TYPES_RT_H
  8589. +
  8590. +#ifndef __LINUX_SPINLOCK_TYPES_H
  8591. +#error "Do not include directly. Include spinlock_types.h instead"
  8592. +#endif
  8593. +
  8594. +/*
  8595. + * rwlocks - rtmutex which allows single reader recursion
  8596. + */
  8597. +typedef struct {
  8598. + struct rt_mutex lock;
  8599. + int read_depth;
  8600. + unsigned int break_lock;
  8601. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8602. + struct lockdep_map dep_map;
  8603. +#endif
  8604. +} rwlock_t;
  8605. +
  8606. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8607. +# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }
  8608. +#else
  8609. +# define RW_DEP_MAP_INIT(lockname)
  8610. +#endif
  8611. +
  8612. +#define __RW_LOCK_UNLOCKED(name) \
  8613. + { .lock = __RT_MUTEX_INITIALIZER_SAVE_STATE(name.lock), \
  8614. + RW_DEP_MAP_INIT(name) }
  8615. +
  8616. +#define DEFINE_RWLOCK(name) \
  8617. + rwlock_t name = __RW_LOCK_UNLOCKED(name)
  8618. +
  8619. +#endif
  8620. diff -Nur linux-4.6.6.orig/include/linux/rwsem.h linux-4.6.6/include/linux/rwsem.h
  8621. --- linux-4.6.6.orig/include/linux/rwsem.h 2016-08-10 12:54:57.000000000 +0200
  8622. +++ linux-4.6.6/include/linux/rwsem.h 2016-08-22 21:27:47.333887985 +0200
  8623. @@ -18,6 +18,10 @@
  8624. #include <linux/osq_lock.h>
  8625. #endif
  8626. +#ifdef CONFIG_PREEMPT_RT_FULL
  8627. +#include <linux/rwsem_rt.h>
  8628. +#else /* PREEMPT_RT_FULL */
  8629. +
  8630. struct rw_semaphore;
  8631. #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
  8632. @@ -177,4 +181,6 @@
  8633. # define up_read_non_owner(sem) up_read(sem)
  8634. #endif
  8635. +#endif /* !PREEMPT_RT_FULL */
  8636. +
  8637. #endif /* _LINUX_RWSEM_H */
  8638. diff -Nur linux-4.6.6.orig/include/linux/rwsem_rt.h linux-4.6.6/include/linux/rwsem_rt.h
  8639. --- linux-4.6.6.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100
  8640. +++ linux-4.6.6/include/linux/rwsem_rt.h 2016-08-22 21:27:47.333887985 +0200
  8641. @@ -0,0 +1,152 @@
  8642. +#ifndef _LINUX_RWSEM_RT_H
  8643. +#define _LINUX_RWSEM_RT_H
  8644. +
  8645. +#ifndef _LINUX_RWSEM_H
  8646. +#error "Include rwsem.h"
  8647. +#endif
  8648. +
  8649. +/*
  8650. + * RW-semaphores are a spinlock plus a reader-depth count.
  8651. + *
  8652. + * Note that the semantics are different from the usual
  8653. + * Linux rw-sems, in PREEMPT_RT mode we do not allow
  8654. + * multiple readers to hold the lock at once, we only allow
  8655. + * a read-lock owner to read-lock recursively. This is
  8656. + * better for latency, makes the implementation inherently
  8657. + * fair and makes it simpler as well.
  8658. + */
  8659. +
  8660. +#include <linux/rtmutex.h>
  8661. +
  8662. +struct rw_semaphore {
  8663. + struct rt_mutex lock;
  8664. + int read_depth;
  8665. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8666. + struct lockdep_map dep_map;
  8667. +#endif
  8668. +};
  8669. +
  8670. +#define __RWSEM_INITIALIZER(name) \
  8671. + { .lock = __RT_MUTEX_INITIALIZER(name.lock), \
  8672. + RW_DEP_MAP_INIT(name) }
  8673. +
  8674. +#define DECLARE_RWSEM(lockname) \
  8675. + struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname)
  8676. +
  8677. +extern void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name,
  8678. + struct lock_class_key *key);
  8679. +
  8680. +#define __rt_init_rwsem(sem, name, key) \
  8681. + do { \
  8682. + rt_mutex_init(&(sem)->lock); \
  8683. + __rt_rwsem_init((sem), (name), (key));\
  8684. + } while (0)
  8685. +
  8686. +#define __init_rwsem(sem, name, key) __rt_init_rwsem(sem, name, key)
  8687. +
  8688. +# define rt_init_rwsem(sem) \
  8689. +do { \
  8690. + static struct lock_class_key __key; \
  8691. + \
  8692. + __rt_init_rwsem((sem), #sem, &__key); \
  8693. +} while (0)
  8694. +
  8695. +extern void rt_down_write(struct rw_semaphore *rwsem);
  8696. +extern void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass);
  8697. +extern void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass);
  8698. +extern void rt_down_write_nested_lock(struct rw_semaphore *rwsem,
  8699. + struct lockdep_map *nest);
  8700. +extern void rt__down_read(struct rw_semaphore *rwsem);
  8701. +extern void rt_down_read(struct rw_semaphore *rwsem);
  8702. +extern int rt_down_write_trylock(struct rw_semaphore *rwsem);
  8703. +extern int rt__down_read_trylock(struct rw_semaphore *rwsem);
  8704. +extern int rt_down_read_trylock(struct rw_semaphore *rwsem);
  8705. +extern void __rt_up_read(struct rw_semaphore *rwsem);
  8706. +extern void rt_up_read(struct rw_semaphore *rwsem);
  8707. +extern void rt_up_write(struct rw_semaphore *rwsem);
  8708. +extern void rt_downgrade_write(struct rw_semaphore *rwsem);
  8709. +
  8710. +#define init_rwsem(sem) rt_init_rwsem(sem)
  8711. +#define rwsem_is_locked(s) rt_mutex_is_locked(&(s)->lock)
  8712. +
  8713. +static inline int rwsem_is_contended(struct rw_semaphore *sem)
  8714. +{
  8715. + /* rt_mutex_has_waiters() */
  8716. + return !RB_EMPTY_ROOT(&sem->lock.waiters);
  8717. +}
  8718. +
  8719. +static inline void __down_read(struct rw_semaphore *sem)
  8720. +{
  8721. + rt__down_read(sem);
  8722. +}
  8723. +
  8724. +static inline void down_read(struct rw_semaphore *sem)
  8725. +{
  8726. + rt_down_read(sem);
  8727. +}
  8728. +
  8729. +static inline int __down_read_trylock(struct rw_semaphore *sem)
  8730. +{
  8731. + return rt__down_read_trylock(sem);
  8732. +}
  8733. +
  8734. +static inline int down_read_trylock(struct rw_semaphore *sem)
  8735. +{
  8736. + return rt_down_read_trylock(sem);
  8737. +}
  8738. +
  8739. +static inline void down_write(struct rw_semaphore *sem)
  8740. +{
  8741. + rt_down_write(sem);
  8742. +}
  8743. +
  8744. +static inline int down_write_trylock(struct rw_semaphore *sem)
  8745. +{
  8746. + return rt_down_write_trylock(sem);
  8747. +}
  8748. +
  8749. +static inline void __up_read(struct rw_semaphore *sem)
  8750. +{
  8751. + __rt_up_read(sem);
  8752. +}
  8753. +
  8754. +static inline void up_read(struct rw_semaphore *sem)
  8755. +{
  8756. + rt_up_read(sem);
  8757. +}
  8758. +
  8759. +static inline void up_write(struct rw_semaphore *sem)
  8760. +{
  8761. + rt_up_write(sem);
  8762. +}
  8763. +
  8764. +static inline void downgrade_write(struct rw_semaphore *sem)
  8765. +{
  8766. + rt_downgrade_write(sem);
  8767. +}
  8768. +
  8769. +static inline void down_read_nested(struct rw_semaphore *sem, int subclass)
  8770. +{
  8771. + return rt_down_read_nested(sem, subclass);
  8772. +}
  8773. +
  8774. +static inline void down_write_nested(struct rw_semaphore *sem, int subclass)
  8775. +{
  8776. + rt_down_write_nested(sem, subclass);
  8777. +}
  8778. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8779. +static inline void down_write_nest_lock(struct rw_semaphore *sem,
  8780. + struct rw_semaphore *nest_lock)
  8781. +{
  8782. + rt_down_write_nested_lock(sem, &nest_lock->dep_map);
  8783. +}
  8784. +
  8785. +#else
  8786. +
  8787. +static inline void down_write_nest_lock(struct rw_semaphore *sem,
  8788. + struct rw_semaphore *nest_lock)
  8789. +{
  8790. + rt_down_write_nested_lock(sem, NULL);
  8791. +}
  8792. +#endif
  8793. +#endif
  8794. diff -Nur linux-4.6.6.orig/include/linux/sched.h linux-4.6.6/include/linux/sched.h
  8795. --- linux-4.6.6.orig/include/linux/sched.h 2016-08-10 12:54:57.000000000 +0200
  8796. +++ linux-4.6.6/include/linux/sched.h 2016-08-22 21:27:47.333887985 +0200
  8797. @@ -26,6 +26,7 @@
  8798. #include <linux/nodemask.h>
  8799. #include <linux/mm_types.h>
  8800. #include <linux/preempt.h>
  8801. +#include <asm/kmap_types.h>
  8802. #include <asm/page.h>
  8803. #include <asm/ptrace.h>
  8804. @@ -241,10 +242,7 @@
  8805. TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
  8806. __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
  8807. -#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
  8808. #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
  8809. -#define task_is_stopped_or_traced(task) \
  8810. - ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
  8811. #define task_contributes_to_load(task) \
  8812. ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
  8813. (task->flags & PF_FROZEN) == 0 && \
  8814. @@ -310,6 +308,11 @@
  8815. #endif
  8816. +#define __set_current_state_no_track(state_value) \
  8817. + do { current->state = (state_value); } while (0)
  8818. +#define set_current_state_no_track(state_value) \
  8819. + set_mb(current->state, (state_value))
  8820. +
  8821. /* Task command name length */
  8822. #define TASK_COMM_LEN 16
  8823. @@ -981,8 +984,18 @@
  8824. struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
  8825. extern void wake_q_add(struct wake_q_head *head,
  8826. - struct task_struct *task);
  8827. -extern void wake_up_q(struct wake_q_head *head);
  8828. + struct task_struct *task);
  8829. +extern void __wake_up_q(struct wake_q_head *head, bool sleeper);
  8830. +
  8831. +static inline void wake_up_q(struct wake_q_head *head)
  8832. +{
  8833. + __wake_up_q(head, false);
  8834. +}
  8835. +
  8836. +static inline void wake_up_q_sleeper(struct wake_q_head *head)
  8837. +{
  8838. + __wake_up_q(head, true);
  8839. +}
  8840. /*
  8841. * sched-domains (multiprocessor balancing) declarations:
  8842. @@ -1393,6 +1406,7 @@
  8843. struct task_struct {
  8844. volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
  8845. + volatile long saved_state; /* saved state for "spinlock sleepers" */
  8846. void *stack;
  8847. atomic_t usage;
  8848. unsigned int flags; /* per process flags, defined below */
  8849. @@ -1429,6 +1443,12 @@
  8850. #endif
  8851. unsigned int policy;
  8852. +#ifdef CONFIG_PREEMPT_RT_FULL
  8853. + int migrate_disable;
  8854. +# ifdef CONFIG_SCHED_DEBUG
  8855. + int migrate_disable_atomic;
  8856. +# endif
  8857. +#endif
  8858. int nr_cpus_allowed;
  8859. cpumask_t cpus_allowed;
  8860. @@ -1559,6 +1579,9 @@
  8861. struct task_cputime cputime_expires;
  8862. struct list_head cpu_timers[3];
  8863. +#ifdef CONFIG_PREEMPT_RT_BASE
  8864. + struct task_struct *posix_timer_list;
  8865. +#endif
  8866. /* process credentials */
  8867. const struct cred __rcu *real_cred; /* objective and real subjective task
  8868. @@ -1589,10 +1612,15 @@
  8869. /* signal handlers */
  8870. struct signal_struct *signal;
  8871. struct sighand_struct *sighand;
  8872. + struct sigqueue *sigqueue_cache;
  8873. sigset_t blocked, real_blocked;
  8874. sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */
  8875. struct sigpending pending;
  8876. +#ifdef CONFIG_PREEMPT_RT_FULL
  8877. + /* TODO: move me into ->restart_block ? */
  8878. + struct siginfo forced_info;
  8879. +#endif
  8880. unsigned long sas_ss_sp;
  8881. size_t sas_ss_size;
  8882. @@ -1820,6 +1848,12 @@
  8883. /* bitmask and counter of trace recursion */
  8884. unsigned long trace_recursion;
  8885. #endif /* CONFIG_TRACING */
  8886. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  8887. + u64 preempt_timestamp_hist;
  8888. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  8889. + long timer_offset;
  8890. +#endif
  8891. +#endif
  8892. #ifdef CONFIG_KCOV
  8893. /* Coverage collection mode enabled for this task (0 if disabled). */
  8894. enum kcov_mode kcov_mode;
  8895. @@ -1845,9 +1879,23 @@
  8896. unsigned int sequential_io;
  8897. unsigned int sequential_io_avg;
  8898. #endif
  8899. +#ifdef CONFIG_PREEMPT_RT_BASE
  8900. + struct rcu_head put_rcu;
  8901. + int softirq_nestcnt;
  8902. + unsigned int softirqs_raised;
  8903. +#endif
  8904. +#ifdef CONFIG_PREEMPT_RT_FULL
  8905. +# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32
  8906. + int kmap_idx;
  8907. + pte_t kmap_pte[KM_TYPE_NR];
  8908. +# endif
  8909. +#endif
  8910. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  8911. unsigned long task_state_change;
  8912. #endif
  8913. +#ifdef CONFIG_PREEMPT_RT_FULL
  8914. + int xmit_recursion;
  8915. +#endif
  8916. int pagefault_disabled;
  8917. #ifdef CONFIG_MMU
  8918. struct task_struct *oom_reaper_list;
  8919. @@ -1868,9 +1916,6 @@
  8920. # define arch_task_struct_size (sizeof(struct task_struct))
  8921. #endif
  8922. -/* Future-safe accessor for struct task_struct's cpus_allowed. */
  8923. -#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
  8924. -
  8925. #define TNF_MIGRATED 0x01
  8926. #define TNF_NO_GROUP 0x02
  8927. #define TNF_SHARED 0x04
  8928. @@ -2060,6 +2105,15 @@
  8929. extern void free_task(struct task_struct *tsk);
  8930. #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
  8931. +#ifdef CONFIG_PREEMPT_RT_BASE
  8932. +extern void __put_task_struct_cb(struct rcu_head *rhp);
  8933. +
  8934. +static inline void put_task_struct(struct task_struct *t)
  8935. +{
  8936. + if (atomic_dec_and_test(&t->usage))
  8937. + call_rcu(&t->put_rcu, __put_task_struct_cb);
  8938. +}
  8939. +#else
  8940. extern void __put_task_struct(struct task_struct *t);
  8941. static inline void put_task_struct(struct task_struct *t)
  8942. @@ -2067,6 +2121,7 @@
  8943. if (atomic_dec_and_test(&t->usage))
  8944. __put_task_struct(t);
  8945. }
  8946. +#endif
  8947. #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
  8948. extern void task_cputime(struct task_struct *t,
  8949. @@ -2105,6 +2160,7 @@
  8950. /*
  8951. * Per process flags
  8952. */
  8953. +#define PF_IN_SOFTIRQ 0x00000001 /* Task is serving softirq */
  8954. #define PF_EXITING 0x00000004 /* getting shut down */
  8955. #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */
  8956. #define PF_VCPU 0x00000010 /* I'm a virtual CPU */
  8957. @@ -2269,6 +2325,10 @@
  8958. extern int set_cpus_allowed_ptr(struct task_struct *p,
  8959. const struct cpumask *new_mask);
  8960. +int migrate_me(void);
  8961. +void tell_sched_cpu_down_begin(int cpu);
  8962. +void tell_sched_cpu_down_done(int cpu);
  8963. +
  8964. #else
  8965. static inline void do_set_cpus_allowed(struct task_struct *p,
  8966. const struct cpumask *new_mask)
  8967. @@ -2281,6 +2341,9 @@
  8968. return -EINVAL;
  8969. return 0;
  8970. }
  8971. +static inline int migrate_me(void) { return 0; }
  8972. +static inline void tell_sched_cpu_down_begin(int cpu) { }
  8973. +static inline void tell_sched_cpu_down_done(int cpu) { }
  8974. #endif
  8975. #ifdef CONFIG_NO_HZ_COMMON
  8976. @@ -2487,6 +2550,7 @@
  8977. extern int wake_up_state(struct task_struct *tsk, unsigned int state);
  8978. extern int wake_up_process(struct task_struct *tsk);
  8979. +extern int wake_up_lock_sleeper(struct task_struct * tsk);
  8980. extern void wake_up_new_task(struct task_struct *tsk);
  8981. #ifdef CONFIG_SMP
  8982. extern void kick_process(struct task_struct *tsk);
  8983. @@ -2610,12 +2674,24 @@
  8984. /* mmdrop drops the mm and the page tables */
  8985. extern void __mmdrop(struct mm_struct *);
  8986. +
  8987. static inline void mmdrop(struct mm_struct * mm)
  8988. {
  8989. if (unlikely(atomic_dec_and_test(&mm->mm_count)))
  8990. __mmdrop(mm);
  8991. }
  8992. +#ifdef CONFIG_PREEMPT_RT_BASE
  8993. +extern void __mmdrop_delayed(struct rcu_head *rhp);
  8994. +static inline void mmdrop_delayed(struct mm_struct *mm)
  8995. +{
  8996. + if (atomic_dec_and_test(&mm->mm_count))
  8997. + call_rcu(&mm->delayed_drop, __mmdrop_delayed);
  8998. +}
  8999. +#else
  9000. +# define mmdrop_delayed(mm) mmdrop(mm)
  9001. +#endif
  9002. +
  9003. /* mmput gets rid of the mappings and all user-space */
  9004. extern void mmput(struct mm_struct *);
  9005. /* Grab a reference to a task's mm, if it is not already going away */
  9006. @@ -2933,6 +3009,43 @@
  9007. return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
  9008. }
  9009. +#ifdef CONFIG_PREEMPT_LAZY
  9010. +static inline void set_tsk_need_resched_lazy(struct task_struct *tsk)
  9011. +{
  9012. + set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
  9013. +}
  9014. +
  9015. +static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk)
  9016. +{
  9017. + clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
  9018. +}
  9019. +
  9020. +static inline int test_tsk_need_resched_lazy(struct task_struct *tsk)
  9021. +{
  9022. + return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY));
  9023. +}
  9024. +
  9025. +static inline int need_resched_lazy(void)
  9026. +{
  9027. + return test_thread_flag(TIF_NEED_RESCHED_LAZY);
  9028. +}
  9029. +
  9030. +static inline int need_resched_now(void)
  9031. +{
  9032. + return test_thread_flag(TIF_NEED_RESCHED);
  9033. +}
  9034. +
  9035. +#else
  9036. +static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { }
  9037. +static inline int need_resched_lazy(void) { return 0; }
  9038. +
  9039. +static inline int need_resched_now(void)
  9040. +{
  9041. + return test_thread_flag(TIF_NEED_RESCHED);
  9042. +}
  9043. +
  9044. +#endif
  9045. +
  9046. static inline int restart_syscall(void)
  9047. {
  9048. set_tsk_thread_flag(current, TIF_SIGPENDING);
  9049. @@ -2964,6 +3077,51 @@
  9050. return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
  9051. }
  9052. +static inline bool __task_is_stopped_or_traced(struct task_struct *task)
  9053. +{
  9054. + if (task->state & (__TASK_STOPPED | __TASK_TRACED))
  9055. + return true;
  9056. +#ifdef CONFIG_PREEMPT_RT_FULL
  9057. + if (task->saved_state & (__TASK_STOPPED | __TASK_TRACED))
  9058. + return true;
  9059. +#endif
  9060. + return false;
  9061. +}
  9062. +
  9063. +static inline bool task_is_stopped_or_traced(struct task_struct *task)
  9064. +{
  9065. + bool traced_stopped;
  9066. +
  9067. +#ifdef CONFIG_PREEMPT_RT_FULL
  9068. + unsigned long flags;
  9069. +
  9070. + raw_spin_lock_irqsave(&task->pi_lock, flags);
  9071. + traced_stopped = __task_is_stopped_or_traced(task);
  9072. + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  9073. +#else
  9074. + traced_stopped = __task_is_stopped_or_traced(task);
  9075. +#endif
  9076. + return traced_stopped;
  9077. +}
  9078. +
  9079. +static inline bool task_is_traced(struct task_struct *task)
  9080. +{
  9081. + bool traced = false;
  9082. +
  9083. + if (task->state & __TASK_TRACED)
  9084. + return true;
  9085. +#ifdef CONFIG_PREEMPT_RT_FULL
  9086. + /* in case the task is sleeping on tasklist_lock */
  9087. + raw_spin_lock_irq(&task->pi_lock);
  9088. + if (task->state & __TASK_TRACED)
  9089. + traced = true;
  9090. + else if (task->saved_state & __TASK_TRACED)
  9091. + traced = true;
  9092. + raw_spin_unlock_irq(&task->pi_lock);
  9093. +#endif
  9094. + return traced;
  9095. +}
  9096. +
  9097. /*
  9098. * cond_resched() and cond_resched_lock(): latency reduction via
  9099. * explicit rescheduling in places that are safe. The return
  9100. @@ -2985,12 +3143,16 @@
  9101. __cond_resched_lock(lock); \
  9102. })
  9103. +#ifndef CONFIG_PREEMPT_RT_FULL
  9104. extern int __cond_resched_softirq(void);
  9105. #define cond_resched_softirq() ({ \
  9106. ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \
  9107. __cond_resched_softirq(); \
  9108. })
  9109. +#else
  9110. +# define cond_resched_softirq() cond_resched()
  9111. +#endif
  9112. static inline void cond_resched_rcu(void)
  9113. {
  9114. @@ -3152,6 +3314,31 @@
  9115. #endif /* CONFIG_SMP */
  9116. +static inline int __migrate_disabled(struct task_struct *p)
  9117. +{
  9118. +#ifdef CONFIG_PREEMPT_RT_FULL
  9119. + return p->migrate_disable;
  9120. +#else
  9121. + return 0;
  9122. +#endif
  9123. +}
  9124. +
  9125. +/* Future-safe accessor for struct task_struct's cpus_allowed. */
  9126. +static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p)
  9127. +{
  9128. + if (__migrate_disabled(p))
  9129. + return cpumask_of(task_cpu(p));
  9130. +
  9131. + return &p->cpus_allowed;
  9132. +}
  9133. +
  9134. +static inline int tsk_nr_cpus_allowed(struct task_struct *p)
  9135. +{
  9136. + if (__migrate_disabled(p))
  9137. + return 1;
  9138. + return p->nr_cpus_allowed;
  9139. +}
  9140. +
  9141. extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
  9142. extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
  9143. diff -Nur linux-4.6.6.orig/include/linux/seqlock.h linux-4.6.6/include/linux/seqlock.h
  9144. --- linux-4.6.6.orig/include/linux/seqlock.h 2016-08-10 12:54:57.000000000 +0200
  9145. +++ linux-4.6.6/include/linux/seqlock.h 2016-08-22 21:27:47.337888238 +0200
  9146. @@ -220,20 +220,30 @@
  9147. return __read_seqcount_retry(s, start);
  9148. }
  9149. -
  9150. -
  9151. -static inline void raw_write_seqcount_begin(seqcount_t *s)
  9152. +static inline void __raw_write_seqcount_begin(seqcount_t *s)
  9153. {
  9154. s->sequence++;
  9155. smp_wmb();
  9156. }
  9157. -static inline void raw_write_seqcount_end(seqcount_t *s)
  9158. +static inline void raw_write_seqcount_begin(seqcount_t *s)
  9159. +{
  9160. + preempt_disable_rt();
  9161. + __raw_write_seqcount_begin(s);
  9162. +}
  9163. +
  9164. +static inline void __raw_write_seqcount_end(seqcount_t *s)
  9165. {
  9166. smp_wmb();
  9167. s->sequence++;
  9168. }
  9169. +static inline void raw_write_seqcount_end(seqcount_t *s)
  9170. +{
  9171. + __raw_write_seqcount_end(s);
  9172. + preempt_enable_rt();
  9173. +}
  9174. +
  9175. /**
  9176. * raw_write_seqcount_barrier - do a seq write barrier
  9177. * @s: pointer to seqcount_t
  9178. @@ -425,10 +435,32 @@
  9179. /*
  9180. * Read side functions for starting and finalizing a read side section.
  9181. */
  9182. +#ifndef CONFIG_PREEMPT_RT_FULL
  9183. static inline unsigned read_seqbegin(const seqlock_t *sl)
  9184. {
  9185. return read_seqcount_begin(&sl->seqcount);
  9186. }
  9187. +#else
  9188. +/*
  9189. + * Starvation safe read side for RT
  9190. + */
  9191. +static inline unsigned read_seqbegin(seqlock_t *sl)
  9192. +{
  9193. + unsigned ret;
  9194. +
  9195. +repeat:
  9196. + ret = ACCESS_ONCE(sl->seqcount.sequence);
  9197. + if (unlikely(ret & 1)) {
  9198. + /*
  9199. + * Take the lock and let the writer proceed (i.e. evtl
  9200. + * boost it), otherwise we could loop here forever.
  9201. + */
  9202. + spin_unlock_wait(&sl->lock);
  9203. + goto repeat;
  9204. + }
  9205. + return ret;
  9206. +}
  9207. +#endif
  9208. static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
  9209. {
  9210. @@ -443,36 +475,36 @@
  9211. static inline void write_seqlock(seqlock_t *sl)
  9212. {
  9213. spin_lock(&sl->lock);
  9214. - write_seqcount_begin(&sl->seqcount);
  9215. + __raw_write_seqcount_begin(&sl->seqcount);
  9216. }
  9217. static inline void write_sequnlock(seqlock_t *sl)
  9218. {
  9219. - write_seqcount_end(&sl->seqcount);
  9220. + __raw_write_seqcount_end(&sl->seqcount);
  9221. spin_unlock(&sl->lock);
  9222. }
  9223. static inline void write_seqlock_bh(seqlock_t *sl)
  9224. {
  9225. spin_lock_bh(&sl->lock);
  9226. - write_seqcount_begin(&sl->seqcount);
  9227. + __raw_write_seqcount_begin(&sl->seqcount);
  9228. }
  9229. static inline void write_sequnlock_bh(seqlock_t *sl)
  9230. {
  9231. - write_seqcount_end(&sl->seqcount);
  9232. + __raw_write_seqcount_end(&sl->seqcount);
  9233. spin_unlock_bh(&sl->lock);
  9234. }
  9235. static inline void write_seqlock_irq(seqlock_t *sl)
  9236. {
  9237. spin_lock_irq(&sl->lock);
  9238. - write_seqcount_begin(&sl->seqcount);
  9239. + __raw_write_seqcount_begin(&sl->seqcount);
  9240. }
  9241. static inline void write_sequnlock_irq(seqlock_t *sl)
  9242. {
  9243. - write_seqcount_end(&sl->seqcount);
  9244. + __raw_write_seqcount_end(&sl->seqcount);
  9245. spin_unlock_irq(&sl->lock);
  9246. }
  9247. @@ -481,7 +513,7 @@
  9248. unsigned long flags;
  9249. spin_lock_irqsave(&sl->lock, flags);
  9250. - write_seqcount_begin(&sl->seqcount);
  9251. + __raw_write_seqcount_begin(&sl->seqcount);
  9252. return flags;
  9253. }
  9254. @@ -491,7 +523,7 @@
  9255. static inline void
  9256. write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
  9257. {
  9258. - write_seqcount_end(&sl->seqcount);
  9259. + __raw_write_seqcount_end(&sl->seqcount);
  9260. spin_unlock_irqrestore(&sl->lock, flags);
  9261. }
  9262. diff -Nur linux-4.6.6.orig/include/linux/signal.h linux-4.6.6/include/linux/signal.h
  9263. --- linux-4.6.6.orig/include/linux/signal.h 2016-08-10 12:54:57.000000000 +0200
  9264. +++ linux-4.6.6/include/linux/signal.h 2016-08-22 21:27:47.337888238 +0200
  9265. @@ -233,6 +233,7 @@
  9266. }
  9267. extern void flush_sigqueue(struct sigpending *queue);
  9268. +extern void flush_task_sigqueue(struct task_struct *tsk);
  9269. /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */
  9270. static inline int valid_signal(unsigned long sig)
  9271. diff -Nur linux-4.6.6.orig/include/linux/skbuff.h linux-4.6.6/include/linux/skbuff.h
  9272. --- linux-4.6.6.orig/include/linux/skbuff.h 2016-08-10 12:54:57.000000000 +0200
  9273. +++ linux-4.6.6/include/linux/skbuff.h 2016-08-22 21:27:47.337888238 +0200
  9274. @@ -283,6 +283,7 @@
  9275. __u32 qlen;
  9276. spinlock_t lock;
  9277. + raw_spinlock_t raw_lock;
  9278. };
  9279. struct sk_buff;
  9280. @@ -1538,6 +1539,12 @@
  9281. __skb_queue_head_init(list);
  9282. }
  9283. +static inline void skb_queue_head_init_raw(struct sk_buff_head *list)
  9284. +{
  9285. + raw_spin_lock_init(&list->raw_lock);
  9286. + __skb_queue_head_init(list);
  9287. +}
  9288. +
  9289. static inline void skb_queue_head_init_class(struct sk_buff_head *list,
  9290. struct lock_class_key *class)
  9291. {
  9292. diff -Nur linux-4.6.6.orig/include/linux/smp.h linux-4.6.6/include/linux/smp.h
  9293. --- linux-4.6.6.orig/include/linux/smp.h 2016-08-10 12:54:57.000000000 +0200
  9294. +++ linux-4.6.6/include/linux/smp.h 2016-08-22 21:27:47.341888498 +0200
  9295. @@ -185,6 +185,9 @@
  9296. #define get_cpu() ({ preempt_disable(); smp_processor_id(); })
  9297. #define put_cpu() preempt_enable()
  9298. +#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); })
  9299. +#define put_cpu_light() migrate_enable()
  9300. +
  9301. /*
  9302. * Callback to arch code if there's nosmp or maxcpus=0 on the
  9303. * boot command line:
  9304. diff -Nur linux-4.6.6.orig/include/linux/spinlock_api_smp.h linux-4.6.6/include/linux/spinlock_api_smp.h
  9305. --- linux-4.6.6.orig/include/linux/spinlock_api_smp.h 2016-08-10 12:54:57.000000000 +0200
  9306. +++ linux-4.6.6/include/linux/spinlock_api_smp.h 2016-08-22 21:27:47.341888498 +0200
  9307. @@ -189,6 +189,8 @@
  9308. return 0;
  9309. }
  9310. -#include <linux/rwlock_api_smp.h>
  9311. +#ifndef CONFIG_PREEMPT_RT_FULL
  9312. +# include <linux/rwlock_api_smp.h>
  9313. +#endif
  9314. #endif /* __LINUX_SPINLOCK_API_SMP_H */
  9315. diff -Nur linux-4.6.6.orig/include/linux/spinlock.h linux-4.6.6/include/linux/spinlock.h
  9316. --- linux-4.6.6.orig/include/linux/spinlock.h 2016-08-10 12:54:57.000000000 +0200
  9317. +++ linux-4.6.6/include/linux/spinlock.h 2016-08-22 21:27:47.341888498 +0200
  9318. @@ -271,7 +271,11 @@
  9319. #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock))
  9320. /* Include rwlock functions */
  9321. -#include <linux/rwlock.h>
  9322. +#ifdef CONFIG_PREEMPT_RT_FULL
  9323. +# include <linux/rwlock_rt.h>
  9324. +#else
  9325. +# include <linux/rwlock.h>
  9326. +#endif
  9327. /*
  9328. * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
  9329. @@ -282,6 +286,10 @@
  9330. # include <linux/spinlock_api_up.h>
  9331. #endif
  9332. +#ifdef CONFIG_PREEMPT_RT_FULL
  9333. +# include <linux/spinlock_rt.h>
  9334. +#else /* PREEMPT_RT_FULL */
  9335. +
  9336. /*
  9337. * Map the spin_lock functions to the raw variants for PREEMPT_RT=n
  9338. */
  9339. @@ -416,4 +424,6 @@
  9340. #define atomic_dec_and_lock(atomic, lock) \
  9341. __cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
  9342. +#endif /* !PREEMPT_RT_FULL */
  9343. +
  9344. #endif /* __LINUX_SPINLOCK_H */
  9345. diff -Nur linux-4.6.6.orig/include/linux/spinlock_rt.h linux-4.6.6/include/linux/spinlock_rt.h
  9346. --- linux-4.6.6.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100
  9347. +++ linux-4.6.6/include/linux/spinlock_rt.h 2016-08-22 21:27:47.341888498 +0200
  9348. @@ -0,0 +1,163 @@
  9349. +#ifndef __LINUX_SPINLOCK_RT_H
  9350. +#define __LINUX_SPINLOCK_RT_H
  9351. +
  9352. +#ifndef __LINUX_SPINLOCK_H
  9353. +#error Do not include directly. Use spinlock.h
  9354. +#endif
  9355. +
  9356. +#include <linux/bug.h>
  9357. +
  9358. +extern void
  9359. +__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key);
  9360. +
  9361. +#define spin_lock_init(slock) \
  9362. +do { \
  9363. + static struct lock_class_key __key; \
  9364. + \
  9365. + rt_mutex_init(&(slock)->lock); \
  9366. + __rt_spin_lock_init(slock, #slock, &__key); \
  9367. +} while (0)
  9368. +
  9369. +void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock);
  9370. +void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock);
  9371. +int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock);
  9372. +
  9373. +extern void __lockfunc rt_spin_lock(spinlock_t *lock);
  9374. +extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock);
  9375. +extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass);
  9376. +extern void __lockfunc rt_spin_unlock(spinlock_t *lock);
  9377. +extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock);
  9378. +extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags);
  9379. +extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock);
  9380. +extern int __lockfunc rt_spin_trylock(spinlock_t *lock);
  9381. +extern int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock);
  9382. +
  9383. +/*
  9384. + * lockdep-less calls, for derived types like rwlock:
  9385. + * (for trylock they can use rt_mutex_trylock() directly.
  9386. + */
  9387. +extern void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock);
  9388. +extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock);
  9389. +extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock);
  9390. +extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock);
  9391. +
  9392. +#define spin_lock(lock) rt_spin_lock(lock)
  9393. +
  9394. +#define spin_lock_bh(lock) \
  9395. + do { \
  9396. + local_bh_disable(); \
  9397. + rt_spin_lock(lock); \
  9398. + } while (0)
  9399. +
  9400. +#define spin_lock_irq(lock) spin_lock(lock)
  9401. +
  9402. +#define spin_do_trylock(lock) __cond_lock(lock, rt_spin_trylock(lock))
  9403. +
  9404. +#define spin_trylock(lock) \
  9405. +({ \
  9406. + int __locked; \
  9407. + __locked = spin_do_trylock(lock); \
  9408. + __locked; \
  9409. +})
  9410. +
  9411. +#ifdef CONFIG_LOCKDEP
  9412. +# define spin_lock_nested(lock, subclass) \
  9413. + do { \
  9414. + rt_spin_lock_nested(lock, subclass); \
  9415. + } while (0)
  9416. +
  9417. +#define spin_lock_bh_nested(lock, subclass) \
  9418. + do { \
  9419. + local_bh_disable(); \
  9420. + rt_spin_lock_nested(lock, subclass); \
  9421. + } while (0)
  9422. +
  9423. +# define spin_lock_irqsave_nested(lock, flags, subclass) \
  9424. + do { \
  9425. + typecheck(unsigned long, flags); \
  9426. + flags = 0; \
  9427. + rt_spin_lock_nested(lock, subclass); \
  9428. + } while (0)
  9429. +#else
  9430. +# define spin_lock_nested(lock, subclass) spin_lock(lock)
  9431. +# define spin_lock_bh_nested(lock, subclass) spin_lock_bh(lock)
  9432. +
  9433. +# define spin_lock_irqsave_nested(lock, flags, subclass) \
  9434. + do { \
  9435. + typecheck(unsigned long, flags); \
  9436. + flags = 0; \
  9437. + spin_lock(lock); \
  9438. + } while (0)
  9439. +#endif
  9440. +
  9441. +#define spin_lock_irqsave(lock, flags) \
  9442. + do { \
  9443. + typecheck(unsigned long, flags); \
  9444. + flags = 0; \
  9445. + spin_lock(lock); \
  9446. + } while (0)
  9447. +
  9448. +static inline unsigned long spin_lock_trace_flags(spinlock_t *lock)
  9449. +{
  9450. + unsigned long flags = 0;
  9451. +#ifdef CONFIG_TRACE_IRQFLAGS
  9452. + flags = rt_spin_lock_trace_flags(lock);
  9453. +#else
  9454. + spin_lock(lock); /* lock_local */
  9455. +#endif
  9456. + return flags;
  9457. +}
  9458. +
  9459. +/* FIXME: we need rt_spin_lock_nest_lock */
  9460. +#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0)
  9461. +
  9462. +#define spin_unlock(lock) rt_spin_unlock(lock)
  9463. +
  9464. +#define spin_unlock_bh(lock) \
  9465. + do { \
  9466. + rt_spin_unlock(lock); \
  9467. + local_bh_enable(); \
  9468. + } while (0)
  9469. +
  9470. +#define spin_unlock_irq(lock) spin_unlock(lock)
  9471. +
  9472. +#define spin_unlock_irqrestore(lock, flags) \
  9473. + do { \
  9474. + typecheck(unsigned long, flags); \
  9475. + (void) flags; \
  9476. + spin_unlock(lock); \
  9477. + } while (0)
  9478. +
  9479. +#define spin_trylock_bh(lock) __cond_lock(lock, rt_spin_trylock_bh(lock))
  9480. +#define spin_trylock_irq(lock) spin_trylock(lock)
  9481. +
  9482. +#define spin_trylock_irqsave(lock, flags) \
  9483. + rt_spin_trylock_irqsave(lock, &(flags))
  9484. +
  9485. +#define spin_unlock_wait(lock) rt_spin_unlock_wait(lock)
  9486. +
  9487. +#ifdef CONFIG_GENERIC_LOCKBREAK
  9488. +# define spin_is_contended(lock) ((lock)->break_lock)
  9489. +#else
  9490. +# define spin_is_contended(lock) (((void)(lock), 0))
  9491. +#endif
  9492. +
  9493. +static inline int spin_can_lock(spinlock_t *lock)
  9494. +{
  9495. + return !rt_mutex_is_locked(&lock->lock);
  9496. +}
  9497. +
  9498. +static inline int spin_is_locked(spinlock_t *lock)
  9499. +{
  9500. + return rt_mutex_is_locked(&lock->lock);
  9501. +}
  9502. +
  9503. +static inline void assert_spin_locked(spinlock_t *lock)
  9504. +{
  9505. + BUG_ON(!spin_is_locked(lock));
  9506. +}
  9507. +
  9508. +#define atomic_dec_and_lock(atomic, lock) \
  9509. + atomic_dec_and_spin_lock(atomic, lock)
  9510. +
  9511. +#endif
  9512. diff -Nur linux-4.6.6.orig/include/linux/spinlock_types.h linux-4.6.6/include/linux/spinlock_types.h
  9513. --- linux-4.6.6.orig/include/linux/spinlock_types.h 2016-08-10 12:54:57.000000000 +0200
  9514. +++ linux-4.6.6/include/linux/spinlock_types.h 2016-08-22 21:27:47.341888498 +0200
  9515. @@ -9,80 +9,15 @@
  9516. * Released under the General Public License (GPL).
  9517. */
  9518. -#if defined(CONFIG_SMP)
  9519. -# include <asm/spinlock_types.h>
  9520. -#else
  9521. -# include <linux/spinlock_types_up.h>
  9522. -#endif
  9523. -
  9524. -#include <linux/lockdep.h>
  9525. -
  9526. -typedef struct raw_spinlock {
  9527. - arch_spinlock_t raw_lock;
  9528. -#ifdef CONFIG_GENERIC_LOCKBREAK
  9529. - unsigned int break_lock;
  9530. -#endif
  9531. -#ifdef CONFIG_DEBUG_SPINLOCK
  9532. - unsigned int magic, owner_cpu;
  9533. - void *owner;
  9534. -#endif
  9535. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9536. - struct lockdep_map dep_map;
  9537. -#endif
  9538. -} raw_spinlock_t;
  9539. -
  9540. -#define SPINLOCK_MAGIC 0xdead4ead
  9541. -
  9542. -#define SPINLOCK_OWNER_INIT ((void *)-1L)
  9543. -
  9544. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9545. -# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }
  9546. -#else
  9547. -# define SPIN_DEP_MAP_INIT(lockname)
  9548. -#endif
  9549. +#include <linux/spinlock_types_raw.h>
  9550. -#ifdef CONFIG_DEBUG_SPINLOCK
  9551. -# define SPIN_DEBUG_INIT(lockname) \
  9552. - .magic = SPINLOCK_MAGIC, \
  9553. - .owner_cpu = -1, \
  9554. - .owner = SPINLOCK_OWNER_INIT,
  9555. +#ifndef CONFIG_PREEMPT_RT_FULL
  9556. +# include <linux/spinlock_types_nort.h>
  9557. +# include <linux/rwlock_types.h>
  9558. #else
  9559. -# define SPIN_DEBUG_INIT(lockname)
  9560. +# include <linux/rtmutex.h>
  9561. +# include <linux/spinlock_types_rt.h>
  9562. +# include <linux/rwlock_types_rt.h>
  9563. #endif
  9564. -#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \
  9565. - { \
  9566. - .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \
  9567. - SPIN_DEBUG_INIT(lockname) \
  9568. - SPIN_DEP_MAP_INIT(lockname) }
  9569. -
  9570. -#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \
  9571. - (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname)
  9572. -
  9573. -#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x)
  9574. -
  9575. -typedef struct spinlock {
  9576. - union {
  9577. - struct raw_spinlock rlock;
  9578. -
  9579. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9580. -# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map))
  9581. - struct {
  9582. - u8 __padding[LOCK_PADSIZE];
  9583. - struct lockdep_map dep_map;
  9584. - };
  9585. -#endif
  9586. - };
  9587. -} spinlock_t;
  9588. -
  9589. -#define __SPIN_LOCK_INITIALIZER(lockname) \
  9590. - { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } }
  9591. -
  9592. -#define __SPIN_LOCK_UNLOCKED(lockname) \
  9593. - (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname)
  9594. -
  9595. -#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
  9596. -
  9597. -#include <linux/rwlock_types.h>
  9598. -
  9599. #endif /* __LINUX_SPINLOCK_TYPES_H */
  9600. diff -Nur linux-4.6.6.orig/include/linux/spinlock_types_nort.h linux-4.6.6/include/linux/spinlock_types_nort.h
  9601. --- linux-4.6.6.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100
  9602. +++ linux-4.6.6/include/linux/spinlock_types_nort.h 2016-08-22 21:27:47.341888498 +0200
  9603. @@ -0,0 +1,33 @@
  9604. +#ifndef __LINUX_SPINLOCK_TYPES_NORT_H
  9605. +#define __LINUX_SPINLOCK_TYPES_NORT_H
  9606. +
  9607. +#ifndef __LINUX_SPINLOCK_TYPES_H
  9608. +#error "Do not include directly. Include spinlock_types.h instead"
  9609. +#endif
  9610. +
  9611. +/*
  9612. + * The non RT version maps spinlocks to raw_spinlocks
  9613. + */
  9614. +typedef struct spinlock {
  9615. + union {
  9616. + struct raw_spinlock rlock;
  9617. +
  9618. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9619. +# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map))
  9620. + struct {
  9621. + u8 __padding[LOCK_PADSIZE];
  9622. + struct lockdep_map dep_map;
  9623. + };
  9624. +#endif
  9625. + };
  9626. +} spinlock_t;
  9627. +
  9628. +#define __SPIN_LOCK_INITIALIZER(lockname) \
  9629. + { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } }
  9630. +
  9631. +#define __SPIN_LOCK_UNLOCKED(lockname) \
  9632. + (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname)
  9633. +
  9634. +#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
  9635. +
  9636. +#endif
  9637. diff -Nur linux-4.6.6.orig/include/linux/spinlock_types_raw.h linux-4.6.6/include/linux/spinlock_types_raw.h
  9638. --- linux-4.6.6.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100
  9639. +++ linux-4.6.6/include/linux/spinlock_types_raw.h 2016-08-22 21:27:47.341888498 +0200
  9640. @@ -0,0 +1,56 @@
  9641. +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H
  9642. +#define __LINUX_SPINLOCK_TYPES_RAW_H
  9643. +
  9644. +#if defined(CONFIG_SMP)
  9645. +# include <asm/spinlock_types.h>
  9646. +#else
  9647. +# include <linux/spinlock_types_up.h>
  9648. +#endif
  9649. +
  9650. +#include <linux/lockdep.h>
  9651. +
  9652. +typedef struct raw_spinlock {
  9653. + arch_spinlock_t raw_lock;
  9654. +#ifdef CONFIG_GENERIC_LOCKBREAK
  9655. + unsigned int break_lock;
  9656. +#endif
  9657. +#ifdef CONFIG_DEBUG_SPINLOCK
  9658. + unsigned int magic, owner_cpu;
  9659. + void *owner;
  9660. +#endif
  9661. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9662. + struct lockdep_map dep_map;
  9663. +#endif
  9664. +} raw_spinlock_t;
  9665. +
  9666. +#define SPINLOCK_MAGIC 0xdead4ead
  9667. +
  9668. +#define SPINLOCK_OWNER_INIT ((void *)-1L)
  9669. +
  9670. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9671. +# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }
  9672. +#else
  9673. +# define SPIN_DEP_MAP_INIT(lockname)
  9674. +#endif
  9675. +
  9676. +#ifdef CONFIG_DEBUG_SPINLOCK
  9677. +# define SPIN_DEBUG_INIT(lockname) \
  9678. + .magic = SPINLOCK_MAGIC, \
  9679. + .owner_cpu = -1, \
  9680. + .owner = SPINLOCK_OWNER_INIT,
  9681. +#else
  9682. +# define SPIN_DEBUG_INIT(lockname)
  9683. +#endif
  9684. +
  9685. +#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \
  9686. + { \
  9687. + .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \
  9688. + SPIN_DEBUG_INIT(lockname) \
  9689. + SPIN_DEP_MAP_INIT(lockname) }
  9690. +
  9691. +#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \
  9692. + (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname)
  9693. +
  9694. +#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x)
  9695. +
  9696. +#endif
  9697. diff -Nur linux-4.6.6.orig/include/linux/spinlock_types_rt.h linux-4.6.6/include/linux/spinlock_types_rt.h
  9698. --- linux-4.6.6.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100
  9699. +++ linux-4.6.6/include/linux/spinlock_types_rt.h 2016-08-22 21:27:47.341888498 +0200
  9700. @@ -0,0 +1,48 @@
  9701. +#ifndef __LINUX_SPINLOCK_TYPES_RT_H
  9702. +#define __LINUX_SPINLOCK_TYPES_RT_H
  9703. +
  9704. +#ifndef __LINUX_SPINLOCK_TYPES_H
  9705. +#error "Do not include directly. Include spinlock_types.h instead"
  9706. +#endif
  9707. +
  9708. +#include <linux/cache.h>
  9709. +
  9710. +/*
  9711. + * PREEMPT_RT: spinlocks - an RT mutex plus lock-break field:
  9712. + */
  9713. +typedef struct spinlock {
  9714. + struct rt_mutex lock;
  9715. + unsigned int break_lock;
  9716. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  9717. + struct lockdep_map dep_map;
  9718. +#endif
  9719. +} spinlock_t;
  9720. +
  9721. +#ifdef CONFIG_DEBUG_RT_MUTEXES
  9722. +# define __RT_SPIN_INITIALIZER(name) \
  9723. + { \
  9724. + .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \
  9725. + .save_state = 1, \
  9726. + .file = __FILE__, \
  9727. + .line = __LINE__ , \
  9728. + }
  9729. +#else
  9730. +# define __RT_SPIN_INITIALIZER(name) \
  9731. + { \
  9732. + .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \
  9733. + .save_state = 1, \
  9734. + }
  9735. +#endif
  9736. +
  9737. +/*
  9738. +.wait_list = PLIST_HEAD_INIT_RAW((name).lock.wait_list, (name).lock.wait_lock)
  9739. +*/
  9740. +
  9741. +#define __SPIN_LOCK_UNLOCKED(name) \
  9742. + { .lock = __RT_SPIN_INITIALIZER(name.lock), \
  9743. + SPIN_DEP_MAP_INIT(name) }
  9744. +
  9745. +#define DEFINE_SPINLOCK(name) \
  9746. + spinlock_t name = __SPIN_LOCK_UNLOCKED(name)
  9747. +
  9748. +#endif
  9749. diff -Nur linux-4.6.6.orig/include/linux/srcu.h linux-4.6.6/include/linux/srcu.h
  9750. --- linux-4.6.6.orig/include/linux/srcu.h 2016-08-10 12:54:57.000000000 +0200
  9751. +++ linux-4.6.6/include/linux/srcu.h 2016-08-22 21:27:47.341888498 +0200
  9752. @@ -84,10 +84,10 @@
  9753. void process_srcu(struct work_struct *work);
  9754. -#define __SRCU_STRUCT_INIT(name) \
  9755. +#define __SRCU_STRUCT_INIT(name, pcpu_name) \
  9756. { \
  9757. .completed = -300, \
  9758. - .per_cpu_ref = &name##_srcu_array, \
  9759. + .per_cpu_ref = &pcpu_name, \
  9760. .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \
  9761. .running = false, \
  9762. .batch_queue = RCU_BATCH_INIT(name.batch_queue), \
  9763. @@ -119,7 +119,7 @@
  9764. */
  9765. #define __DEFINE_SRCU(name, is_static) \
  9766. static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\
  9767. - is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)
  9768. + is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name##_srcu_array)
  9769. #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
  9770. #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
  9771. diff -Nur linux-4.6.6.orig/include/linux/suspend.h linux-4.6.6/include/linux/suspend.h
  9772. --- linux-4.6.6.orig/include/linux/suspend.h 2016-08-10 12:54:57.000000000 +0200
  9773. +++ linux-4.6.6/include/linux/suspend.h 2016-08-22 21:27:47.345888751 +0200
  9774. @@ -194,6 +194,12 @@
  9775. void (*end)(void);
  9776. };
  9777. +#if defined(CONFIG_SUSPEND) || defined(CONFIG_HIBERNATION)
  9778. +extern bool pm_in_action;
  9779. +#else
  9780. +# define pm_in_action false
  9781. +#endif
  9782. +
  9783. #ifdef CONFIG_SUSPEND
  9784. /**
  9785. * suspend_set_ops - set platform dependent suspend operations
  9786. diff -Nur linux-4.6.6.orig/include/linux/swait.h linux-4.6.6/include/linux/swait.h
  9787. --- linux-4.6.6.orig/include/linux/swait.h 2016-08-10 12:54:57.000000000 +0200
  9788. +++ linux-4.6.6/include/linux/swait.h 2016-08-22 21:27:47.345888751 +0200
  9789. @@ -87,6 +87,7 @@
  9790. extern void swake_up(struct swait_queue_head *q);
  9791. extern void swake_up_all(struct swait_queue_head *q);
  9792. extern void swake_up_locked(struct swait_queue_head *q);
  9793. +extern void swake_up_all_locked(struct swait_queue_head *q);
  9794. extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
  9795. extern void prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait, int state);
  9796. diff -Nur linux-4.6.6.orig/include/linux/swap.h linux-4.6.6/include/linux/swap.h
  9797. --- linux-4.6.6.orig/include/linux/swap.h 2016-08-10 12:54:57.000000000 +0200
  9798. +++ linux-4.6.6/include/linux/swap.h 2016-08-22 21:27:47.345888751 +0200
  9799. @@ -11,6 +11,7 @@
  9800. #include <linux/fs.h>
  9801. #include <linux/atomic.h>
  9802. #include <linux/page-flags.h>
  9803. +#include <linux/locallock.h>
  9804. #include <asm/page.h>
  9805. struct notifier_block;
  9806. @@ -252,7 +253,8 @@
  9807. void *workingset_eviction(struct address_space *mapping, struct page *page);
  9808. bool workingset_refault(void *shadow);
  9809. void workingset_activation(struct page *page);
  9810. -extern struct list_lru workingset_shadow_nodes;
  9811. +extern struct list_lru __workingset_shadow_nodes;
  9812. +DECLARE_LOCAL_IRQ_LOCK(workingset_shadow_lock);
  9813. static inline unsigned int workingset_node_pages(struct radix_tree_node *node)
  9814. {
  9815. @@ -295,6 +297,7 @@
  9816. /* linux/mm/swap.c */
  9817. +DECLARE_LOCAL_IRQ_LOCK(swapvec_lock);
  9818. extern void lru_cache_add(struct page *);
  9819. extern void lru_cache_add_anon(struct page *page);
  9820. extern void lru_cache_add_file(struct page *page);
  9821. diff -Nur linux-4.6.6.orig/include/linux/swork.h linux-4.6.6/include/linux/swork.h
  9822. --- linux-4.6.6.orig/include/linux/swork.h 1970-01-01 01:00:00.000000000 +0100
  9823. +++ linux-4.6.6/include/linux/swork.h 2016-08-22 21:27:47.345888751 +0200
  9824. @@ -0,0 +1,24 @@
  9825. +#ifndef _LINUX_SWORK_H
  9826. +#define _LINUX_SWORK_H
  9827. +
  9828. +#include <linux/list.h>
  9829. +
  9830. +struct swork_event {
  9831. + struct list_head item;
  9832. + unsigned long flags;
  9833. + void (*func)(struct swork_event *);
  9834. +};
  9835. +
  9836. +static inline void INIT_SWORK(struct swork_event *event,
  9837. + void (*func)(struct swork_event *))
  9838. +{
  9839. + event->flags = 0;
  9840. + event->func = func;
  9841. +}
  9842. +
  9843. +bool swork_queue(struct swork_event *sev);
  9844. +
  9845. +int swork_get(void);
  9846. +void swork_put(void);
  9847. +
  9848. +#endif /* _LINUX_SWORK_H */
  9849. diff -Nur linux-4.6.6.orig/include/linux/thread_info.h linux-4.6.6/include/linux/thread_info.h
  9850. --- linux-4.6.6.orig/include/linux/thread_info.h 2016-08-10 12:54:57.000000000 +0200
  9851. +++ linux-4.6.6/include/linux/thread_info.h 2016-08-22 21:27:47.345888751 +0200
  9852. @@ -103,7 +103,17 @@
  9853. #define test_thread_flag(flag) \
  9854. test_ti_thread_flag(current_thread_info(), flag)
  9855. -#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED)
  9856. +#ifdef CONFIG_PREEMPT_LAZY
  9857. +#define tif_need_resched() (test_thread_flag(TIF_NEED_RESCHED) || \
  9858. + test_thread_flag(TIF_NEED_RESCHED_LAZY))
  9859. +#define tif_need_resched_now() (test_thread_flag(TIF_NEED_RESCHED))
  9860. +#define tif_need_resched_lazy() test_thread_flag(TIF_NEED_RESCHED_LAZY))
  9861. +
  9862. +#else
  9863. +#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED)
  9864. +#define tif_need_resched_now() test_thread_flag(TIF_NEED_RESCHED)
  9865. +#define tif_need_resched_lazy() 0
  9866. +#endif
  9867. #if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK
  9868. /*
  9869. diff -Nur linux-4.6.6.orig/include/linux/timer.h linux-4.6.6/include/linux/timer.h
  9870. --- linux-4.6.6.orig/include/linux/timer.h 2016-08-10 12:54:57.000000000 +0200
  9871. +++ linux-4.6.6/include/linux/timer.h 2016-08-22 21:27:47.345888751 +0200
  9872. @@ -19,7 +19,6 @@
  9873. void (*function)(unsigned long);
  9874. unsigned long data;
  9875. u32 flags;
  9876. - int slack;
  9877. #ifdef CONFIG_TIMER_STATS
  9878. int start_pid;
  9879. @@ -58,11 +57,14 @@
  9880. * workqueue locking issues. It's not meant for executing random crap
  9881. * with interrupts disabled. Abuse is monitored!
  9882. */
  9883. -#define TIMER_CPUMASK 0x0007FFFF
  9884. -#define TIMER_MIGRATING 0x00080000
  9885. +#define TIMER_CPUMASK 0x0003FFFF
  9886. +#define TIMER_MIGRATING 0x00040000
  9887. #define TIMER_BASEMASK (TIMER_CPUMASK | TIMER_MIGRATING)
  9888. -#define TIMER_DEFERRABLE 0x00100000
  9889. +#define TIMER_DEFERRABLE 0x00080000
  9890. +#define TIMER_PINNED 0x00100000
  9891. #define TIMER_IRQSAFE 0x00200000
  9892. +#define TIMER_ARRAYSHIFT 22
  9893. +#define TIMER_ARRAYMASK 0xFFC00000
  9894. #define __TIMER_INITIALIZER(_function, _expires, _data, _flags) { \
  9895. .entry = { .next = TIMER_ENTRY_STATIC }, \
  9896. @@ -70,7 +72,6 @@
  9897. .expires = (_expires), \
  9898. .data = (_data), \
  9899. .flags = (_flags), \
  9900. - .slack = -1, \
  9901. __TIMER_LOCKDEP_MAP_INITIALIZER( \
  9902. __FILE__ ":" __stringify(__LINE__)) \
  9903. }
  9904. @@ -78,9 +79,15 @@
  9905. #define TIMER_INITIALIZER(_function, _expires, _data) \
  9906. __TIMER_INITIALIZER((_function), (_expires), (_data), 0)
  9907. +#define TIMER_PINNED_INITIALIZER(_function, _expires, _data) \
  9908. + __TIMER_INITIALIZER((_function), (_expires), (_data), TIMER_PINNED)
  9909. +
  9910. #define TIMER_DEFERRED_INITIALIZER(_function, _expires, _data) \
  9911. __TIMER_INITIALIZER((_function), (_expires), (_data), TIMER_DEFERRABLE)
  9912. +#define TIMER_PINNED_DEFERRED_INITIALIZER(_function, _expires, _data) \
  9913. + __TIMER_INITIALIZER((_function), (_expires), (_data), TIMER_DEFERRABLE | TIMER_PINNED)
  9914. +
  9915. #define DEFINE_TIMER(_name, _function, _expires, _data) \
  9916. struct timer_list _name = \
  9917. TIMER_INITIALIZER(_function, _expires, _data)
  9918. @@ -124,8 +131,12 @@
  9919. #define init_timer(timer) \
  9920. __init_timer((timer), 0)
  9921. +#define init_timer_pinned(timer) \
  9922. + __init_timer((timer), TIMER_PINNED)
  9923. #define init_timer_deferrable(timer) \
  9924. __init_timer((timer), TIMER_DEFERRABLE)
  9925. +#define init_timer_pinned_deferrable(timer) \
  9926. + __init_timer((timer), TIMER_DEFERRABLE | TIMER_PINNED)
  9927. #define init_timer_on_stack(timer) \
  9928. __init_timer_on_stack((timer), 0)
  9929. @@ -145,10 +156,20 @@
  9930. #define setup_timer(timer, fn, data) \
  9931. __setup_timer((timer), (fn), (data), 0)
  9932. +#define setup_pinned_timer(timer, fn, data) \
  9933. + __setup_timer((timer), (fn), (data), TIMER_PINNED)
  9934. +#define setup_deferrable_timer(timer, fn, data) \
  9935. + __setup_timer((timer), (fn), (data), TIMER_DEFERRABLE)
  9936. +#define setup_pinned_deferrable_timer(timer, fn, data) \
  9937. + __setup_timer((timer), (fn), (data), TIMER_DEFERRABLE | TIMER_PINNED)
  9938. #define setup_timer_on_stack(timer, fn, data) \
  9939. __setup_timer_on_stack((timer), (fn), (data), 0)
  9940. +#define setup_pinned_timer_on_stack(timer, fn, data) \
  9941. + __setup_timer_on_stack((timer), (fn), (data), TIMER_PINNED)
  9942. #define setup_deferrable_timer_on_stack(timer, fn, data) \
  9943. __setup_timer_on_stack((timer), (fn), (data), TIMER_DEFERRABLE)
  9944. +#define setup_pinned_deferrable_timer_on_stack(timer, fn, data) \
  9945. + __setup_timer_on_stack((timer), (fn), (data), TIMER_DEFERRABLE | TIMER_PINNED)
  9946. /**
  9947. * timer_pending - is a timer pending?
  9948. @@ -169,12 +190,7 @@
  9949. extern int del_timer(struct timer_list * timer);
  9950. extern int mod_timer(struct timer_list *timer, unsigned long expires);
  9951. extern int mod_timer_pending(struct timer_list *timer, unsigned long expires);
  9952. -extern int mod_timer_pinned(struct timer_list *timer, unsigned long expires);
  9953. -
  9954. -extern void set_timer_slack(struct timer_list *time, int slack_hz);
  9955. -#define TIMER_NOT_PINNED 0
  9956. -#define TIMER_PINNED 1
  9957. /*
  9958. * The jiffies value which is added to now, when there is no timer
  9959. * in the timer wheel:
  9960. @@ -225,7 +241,7 @@
  9961. extern int try_to_del_timer_sync(struct timer_list *timer);
  9962. -#ifdef CONFIG_SMP
  9963. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  9964. extern int del_timer_sync(struct timer_list *timer);
  9965. #else
  9966. # define del_timer_sync(t) del_timer(t)
  9967. diff -Nur linux-4.6.6.orig/include/linux/trace_events.h linux-4.6.6/include/linux/trace_events.h
  9968. --- linux-4.6.6.orig/include/linux/trace_events.h 2016-08-10 12:54:57.000000000 +0200
  9969. +++ linux-4.6.6/include/linux/trace_events.h 2016-08-22 21:27:47.345888751 +0200
  9970. @@ -56,6 +56,9 @@
  9971. unsigned char flags;
  9972. unsigned char preempt_count;
  9973. int pid;
  9974. + unsigned short migrate_disable;
  9975. + unsigned short padding;
  9976. + unsigned char preempt_lazy_count;
  9977. };
  9978. #define TRACE_EVENT_TYPE_MAX \
  9979. diff -Nur linux-4.6.6.orig/include/linux/uaccess.h linux-4.6.6/include/linux/uaccess.h
  9980. --- linux-4.6.6.orig/include/linux/uaccess.h 2016-08-10 12:54:57.000000000 +0200
  9981. +++ linux-4.6.6/include/linux/uaccess.h 2016-08-22 21:27:47.345888751 +0200
  9982. @@ -24,6 +24,7 @@
  9983. */
  9984. static inline void pagefault_disable(void)
  9985. {
  9986. + migrate_disable();
  9987. pagefault_disabled_inc();
  9988. /*
  9989. * make sure to have issued the store before a pagefault
  9990. @@ -40,6 +41,7 @@
  9991. */
  9992. barrier();
  9993. pagefault_disabled_dec();
  9994. + migrate_enable();
  9995. }
  9996. /*
  9997. diff -Nur linux-4.6.6.orig/include/linux/uprobes.h linux-4.6.6/include/linux/uprobes.h
  9998. --- linux-4.6.6.orig/include/linux/uprobes.h 2016-08-10 12:54:57.000000000 +0200
  9999. +++ linux-4.6.6/include/linux/uprobes.h 2016-08-22 21:27:47.345888751 +0200
  10000. @@ -27,6 +27,7 @@
  10001. #include <linux/errno.h>
  10002. #include <linux/rbtree.h>
  10003. #include <linux/types.h>
  10004. +#include <linux/wait.h>
  10005. struct vm_area_struct;
  10006. struct mm_struct;
  10007. diff -Nur linux-4.6.6.orig/include/linux/vmstat.h linux-4.6.6/include/linux/vmstat.h
  10008. --- linux-4.6.6.orig/include/linux/vmstat.h 2016-08-10 12:54:57.000000000 +0200
  10009. +++ linux-4.6.6/include/linux/vmstat.h 2016-08-22 21:27:47.345888751 +0200
  10010. @@ -33,7 +33,9 @@
  10011. */
  10012. static inline void __count_vm_event(enum vm_event_item item)
  10013. {
  10014. + preempt_disable_rt();
  10015. raw_cpu_inc(vm_event_states.event[item]);
  10016. + preempt_enable_rt();
  10017. }
  10018. static inline void count_vm_event(enum vm_event_item item)
  10019. @@ -43,7 +45,9 @@
  10020. static inline void __count_vm_events(enum vm_event_item item, long delta)
  10021. {
  10022. + preempt_disable_rt();
  10023. raw_cpu_add(vm_event_states.event[item], delta);
  10024. + preempt_enable_rt();
  10025. }
  10026. static inline void count_vm_events(enum vm_event_item item, long delta)
  10027. diff -Nur linux-4.6.6.orig/include/linux/wait.h linux-4.6.6/include/linux/wait.h
  10028. --- linux-4.6.6.orig/include/linux/wait.h 2016-08-10 12:54:57.000000000 +0200
  10029. +++ linux-4.6.6/include/linux/wait.h 2016-08-22 21:27:47.349889011 +0200
  10030. @@ -8,6 +8,7 @@
  10031. #include <linux/spinlock.h>
  10032. #include <asm/current.h>
  10033. #include <uapi/linux/wait.h>
  10034. +#include <linux/atomic.h>
  10035. typedef struct __wait_queue wait_queue_t;
  10036. typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
  10037. diff -Nur linux-4.6.6.orig/include/net/dst.h linux-4.6.6/include/net/dst.h
  10038. --- linux-4.6.6.orig/include/net/dst.h 2016-08-10 12:54:57.000000000 +0200
  10039. +++ linux-4.6.6/include/net/dst.h 2016-08-22 21:27:47.349889011 +0200
  10040. @@ -449,7 +449,7 @@
  10041. static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n,
  10042. struct sk_buff *skb)
  10043. {
  10044. - const struct hh_cache *hh;
  10045. + struct hh_cache *hh;
  10046. if (dst->pending_confirm) {
  10047. unsigned long now = jiffies;
  10048. diff -Nur linux-4.6.6.orig/include/net/neighbour.h linux-4.6.6/include/net/neighbour.h
  10049. --- linux-4.6.6.orig/include/net/neighbour.h 2016-08-10 12:54:57.000000000 +0200
  10050. +++ linux-4.6.6/include/net/neighbour.h 2016-08-22 21:27:47.349889011 +0200
  10051. @@ -446,7 +446,7 @@
  10052. }
  10053. #endif
  10054. -static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb)
  10055. +static inline int neigh_hh_output(struct hh_cache *hh, struct sk_buff *skb)
  10056. {
  10057. unsigned int seq;
  10058. int hh_len;
  10059. @@ -501,7 +501,7 @@
  10060. #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb)
  10061. -static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n,
  10062. +static inline void neigh_ha_snapshot(char *dst, struct neighbour *n,
  10063. const struct net_device *dev)
  10064. {
  10065. unsigned int seq;
  10066. diff -Nur linux-4.6.6.orig/include/net/netns/ipv4.h linux-4.6.6/include/net/netns/ipv4.h
  10067. --- linux-4.6.6.orig/include/net/netns/ipv4.h 2016-08-10 12:54:57.000000000 +0200
  10068. +++ linux-4.6.6/include/net/netns/ipv4.h 2016-08-22 21:27:47.349889011 +0200
  10069. @@ -70,6 +70,7 @@
  10070. int sysctl_icmp_echo_ignore_all;
  10071. int sysctl_icmp_echo_ignore_broadcasts;
  10072. + int sysctl_icmp_echo_sysrq;
  10073. int sysctl_icmp_ignore_bogus_error_responses;
  10074. int sysctl_icmp_ratelimit;
  10075. int sysctl_icmp_ratemask;
  10076. diff -Nur linux-4.6.6.orig/include/trace/events/hist.h linux-4.6.6/include/trace/events/hist.h
  10077. --- linux-4.6.6.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100
  10078. +++ linux-4.6.6/include/trace/events/hist.h 2016-08-22 21:27:47.349889011 +0200
  10079. @@ -0,0 +1,73 @@
  10080. +#undef TRACE_SYSTEM
  10081. +#define TRACE_SYSTEM hist
  10082. +
  10083. +#if !defined(_TRACE_HIST_H) || defined(TRACE_HEADER_MULTI_READ)
  10084. +#define _TRACE_HIST_H
  10085. +
  10086. +#include "latency_hist.h"
  10087. +#include <linux/tracepoint.h>
  10088. +
  10089. +#if !defined(CONFIG_PREEMPT_OFF_HIST) && !defined(CONFIG_INTERRUPT_OFF_HIST)
  10090. +#define trace_preemptirqsoff_hist(a, b)
  10091. +#define trace_preemptirqsoff_hist_rcuidle(a, b)
  10092. +#else
  10093. +TRACE_EVENT(preemptirqsoff_hist,
  10094. +
  10095. + TP_PROTO(int reason, int starthist),
  10096. +
  10097. + TP_ARGS(reason, starthist),
  10098. +
  10099. + TP_STRUCT__entry(
  10100. + __field(int, reason)
  10101. + __field(int, starthist)
  10102. + ),
  10103. +
  10104. + TP_fast_assign(
  10105. + __entry->reason = reason;
  10106. + __entry->starthist = starthist;
  10107. + ),
  10108. +
  10109. + TP_printk("reason=%s starthist=%s", getaction(__entry->reason),
  10110. + __entry->starthist ? "start" : "stop")
  10111. +);
  10112. +#endif
  10113. +
  10114. +#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST
  10115. +#define trace_hrtimer_interrupt(a, b, c, d)
  10116. +#else
  10117. +TRACE_EVENT(hrtimer_interrupt,
  10118. +
  10119. + TP_PROTO(int cpu, long long offset, struct task_struct *curr,
  10120. + struct task_struct *task),
  10121. +
  10122. + TP_ARGS(cpu, offset, curr, task),
  10123. +
  10124. + TP_STRUCT__entry(
  10125. + __field(int, cpu)
  10126. + __field(long long, offset)
  10127. + __array(char, ccomm, TASK_COMM_LEN)
  10128. + __field(int, cprio)
  10129. + __array(char, tcomm, TASK_COMM_LEN)
  10130. + __field(int, tprio)
  10131. + ),
  10132. +
  10133. + TP_fast_assign(
  10134. + __entry->cpu = cpu;
  10135. + __entry->offset = offset;
  10136. + memcpy(__entry->ccomm, curr->comm, TASK_COMM_LEN);
  10137. + __entry->cprio = curr->prio;
  10138. + memcpy(__entry->tcomm, task != NULL ? task->comm : "<none>",
  10139. + task != NULL ? TASK_COMM_LEN : 7);
  10140. + __entry->tprio = task != NULL ? task->prio : -1;
  10141. + ),
  10142. +
  10143. + TP_printk("cpu=%d offset=%lld curr=%s[%d] thread=%s[%d]",
  10144. + __entry->cpu, __entry->offset, __entry->ccomm,
  10145. + __entry->cprio, __entry->tcomm, __entry->tprio)
  10146. +);
  10147. +#endif
  10148. +
  10149. +#endif /* _TRACE_HIST_H */
  10150. +
  10151. +/* This part must be outside protection */
  10152. +#include <trace/define_trace.h>
  10153. diff -Nur linux-4.6.6.orig/include/trace/events/latency_hist.h linux-4.6.6/include/trace/events/latency_hist.h
  10154. --- linux-4.6.6.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100
  10155. +++ linux-4.6.6/include/trace/events/latency_hist.h 2016-08-22 21:27:47.349889011 +0200
  10156. @@ -0,0 +1,29 @@
  10157. +#ifndef _LATENCY_HIST_H
  10158. +#define _LATENCY_HIST_H
  10159. +
  10160. +enum hist_action {
  10161. + IRQS_ON,
  10162. + PREEMPT_ON,
  10163. + TRACE_STOP,
  10164. + IRQS_OFF,
  10165. + PREEMPT_OFF,
  10166. + TRACE_START,
  10167. +};
  10168. +
  10169. +static char *actions[] = {
  10170. + "IRQS_ON",
  10171. + "PREEMPT_ON",
  10172. + "TRACE_STOP",
  10173. + "IRQS_OFF",
  10174. + "PREEMPT_OFF",
  10175. + "TRACE_START",
  10176. +};
  10177. +
  10178. +static inline char *getaction(int action)
  10179. +{
  10180. + if (action >= 0 && action <= sizeof(actions)/sizeof(actions[0]))
  10181. + return actions[action];
  10182. + return "unknown";
  10183. +}
  10184. +
  10185. +#endif /* _LATENCY_HIST_H */
  10186. diff -Nur linux-4.6.6.orig/init/Kconfig linux-4.6.6/init/Kconfig
  10187. --- linux-4.6.6.orig/init/Kconfig 2016-08-10 12:54:57.000000000 +0200
  10188. +++ linux-4.6.6/init/Kconfig 2016-08-22 21:27:47.353889264 +0200
  10189. @@ -494,7 +494,7 @@
  10190. config RCU_EXPERT
  10191. bool "Make expert-level adjustments to RCU configuration"
  10192. - default n
  10193. + default y if PREEMPT_RT_FULL
  10194. help
  10195. This option needs to be enabled if you wish to make
  10196. expert-level adjustments to RCU configuration. By default,
  10197. @@ -610,7 +610,7 @@
  10198. config RCU_FAST_NO_HZ
  10199. bool "Accelerate last non-dyntick-idle CPU's grace periods"
  10200. - depends on NO_HZ_COMMON && SMP && RCU_EXPERT
  10201. + depends on NO_HZ_COMMON && SMP && RCU_EXPERT && !PREEMPT_RT_FULL
  10202. default n
  10203. help
  10204. This option permits CPUs to enter dynticks-idle state even if
  10205. @@ -637,7 +637,7 @@
  10206. config RCU_BOOST
  10207. bool "Enable RCU priority boosting"
  10208. depends on RT_MUTEXES && PREEMPT_RCU && RCU_EXPERT
  10209. - default n
  10210. + default y if PREEMPT_RT_FULL
  10211. help
  10212. This option boosts the priority of preempted RCU readers that
  10213. block the current preemptible RCU grace period for too long.
  10214. @@ -1029,6 +1029,7 @@
  10215. config RT_GROUP_SCHED
  10216. bool "Group scheduling for SCHED_RR/FIFO"
  10217. depends on CGROUP_SCHED
  10218. + depends on !PREEMPT_RT_FULL
  10219. default n
  10220. help
  10221. This feature lets you explicitly allocate real CPU bandwidth
  10222. @@ -1718,6 +1719,7 @@
  10223. config SLAB
  10224. bool "SLAB"
  10225. + depends on !PREEMPT_RT_FULL
  10226. help
  10227. The regular slab allocator that is established and known to work
  10228. well in all environments. It organizes cache hot objects in
  10229. @@ -1736,6 +1738,7 @@
  10230. config SLOB
  10231. depends on EXPERT
  10232. bool "SLOB (Simple Allocator)"
  10233. + depends on !PREEMPT_RT_FULL
  10234. help
  10235. SLOB replaces the stock allocator with a drastically simpler
  10236. allocator. SLOB is generally more space efficient but
  10237. @@ -1745,7 +1748,7 @@
  10238. config SLUB_CPU_PARTIAL
  10239. default y
  10240. - depends on SLUB && SMP
  10241. + depends on SLUB && SMP && !PREEMPT_RT_FULL
  10242. bool "SLUB per cpu partial cache"
  10243. help
  10244. Per cpu partial caches accellerate objects allocation and freeing
  10245. diff -Nur linux-4.6.6.orig/init/main.c linux-4.6.6/init/main.c
  10246. --- linux-4.6.6.orig/init/main.c 2016-08-10 12:54:57.000000000 +0200
  10247. +++ linux-4.6.6/init/main.c 2016-08-22 21:27:47.353889264 +0200
  10248. @@ -507,6 +507,7 @@
  10249. setup_command_line(command_line);
  10250. setup_nr_cpu_ids();
  10251. setup_per_cpu_areas();
  10252. + softirq_early_init();
  10253. boot_cpu_state_init();
  10254. smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
  10255. diff -Nur linux-4.6.6.orig/init/Makefile linux-4.6.6/init/Makefile
  10256. --- linux-4.6.6.orig/init/Makefile 2016-08-10 12:54:57.000000000 +0200
  10257. +++ linux-4.6.6/init/Makefile 2016-08-22 21:27:47.353889264 +0200
  10258. @@ -33,4 +33,4 @@
  10259. include/generated/compile.h: FORCE
  10260. @$($(quiet)chk_compile.h)
  10261. $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \
  10262. - "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)"
  10263. + "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)"
  10264. diff -Nur linux-4.6.6.orig/ipc/msg.c linux-4.6.6/ipc/msg.c
  10265. --- linux-4.6.6.orig/ipc/msg.c 2016-08-10 12:54:57.000000000 +0200
  10266. +++ linux-4.6.6/ipc/msg.c 2016-08-22 21:27:47.353889264 +0200
  10267. @@ -183,20 +183,14 @@
  10268. }
  10269. }
  10270. -static void expunge_all(struct msg_queue *msq, int res)
  10271. +static void expunge_all(struct msg_queue *msq, int res,
  10272. + struct wake_q_head *wake_q)
  10273. {
  10274. struct msg_receiver *msr, *t;
  10275. list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) {
  10276. - msr->r_msg = NULL; /* initialize expunge ordering */
  10277. - wake_up_process(msr->r_tsk);
  10278. - /*
  10279. - * Ensure that the wakeup is visible before setting r_msg as
  10280. - * the receiving end depends on it: either spinning on a nil,
  10281. - * or dealing with -EAGAIN cases. See lockless receive part 1
  10282. - * and 2 in do_msgrcv().
  10283. - */
  10284. - smp_wmb(); /* barrier (B) */
  10285. +
  10286. + wake_q_add(wake_q, msr->r_tsk);
  10287. msr->r_msg = ERR_PTR(res);
  10288. }
  10289. }
  10290. @@ -213,11 +207,13 @@
  10291. {
  10292. struct msg_msg *msg, *t;
  10293. struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm);
  10294. + WAKE_Q(wake_q);
  10295. - expunge_all(msq, -EIDRM);
  10296. + expunge_all(msq, -EIDRM, &wake_q);
  10297. ss_wakeup(&msq->q_senders, 1);
  10298. msg_rmid(ns, msq);
  10299. ipc_unlock_object(&msq->q_perm);
  10300. + wake_up_q(&wake_q);
  10301. rcu_read_unlock();
  10302. list_for_each_entry_safe(msg, t, &msq->q_messages, m_list) {
  10303. @@ -342,6 +338,7 @@
  10304. struct kern_ipc_perm *ipcp;
  10305. struct msqid64_ds uninitialized_var(msqid64);
  10306. struct msg_queue *msq;
  10307. + WAKE_Q(wake_q);
  10308. int err;
  10309. if (cmd == IPC_SET) {
  10310. @@ -389,7 +386,7 @@
  10311. /* sleeping receivers might be excluded by
  10312. * stricter permissions.
  10313. */
  10314. - expunge_all(msq, -EAGAIN);
  10315. + expunge_all(msq, -EAGAIN, &wake_q);
  10316. /* sleeping senders might be able to send
  10317. * due to a larger queue size.
  10318. */
  10319. @@ -402,6 +399,7 @@
  10320. out_unlock0:
  10321. ipc_unlock_object(&msq->q_perm);
  10322. + wake_up_q(&wake_q);
  10323. out_unlock1:
  10324. rcu_read_unlock();
  10325. out_up:
  10326. @@ -566,7 +564,8 @@
  10327. return 0;
  10328. }
  10329. -static inline int pipelined_send(struct msg_queue *msq, struct msg_msg *msg)
  10330. +static inline int pipelined_send(struct msg_queue *msq, struct msg_msg *msg,
  10331. + struct wake_q_head *wake_q)
  10332. {
  10333. struct msg_receiver *msr, *t;
  10334. @@ -577,27 +576,13 @@
  10335. list_del(&msr->r_list);
  10336. if (msr->r_maxsize < msg->m_ts) {
  10337. - /* initialize pipelined send ordering */
  10338. - msr->r_msg = NULL;
  10339. - wake_up_process(msr->r_tsk);
  10340. - /* barrier (B) see barrier comment below */
  10341. - smp_wmb();
  10342. + wake_q_add(wake_q, msr->r_tsk);
  10343. msr->r_msg = ERR_PTR(-E2BIG);
  10344. } else {
  10345. - msr->r_msg = NULL;
  10346. msq->q_lrpid = task_pid_vnr(msr->r_tsk);
  10347. msq->q_rtime = get_seconds();
  10348. - wake_up_process(msr->r_tsk);
  10349. - /*
  10350. - * Ensure that the wakeup is visible before
  10351. - * setting r_msg, as the receiving can otherwise
  10352. - * exit - once r_msg is set, the receiver can
  10353. - * continue. See lockless receive part 1 and 2
  10354. - * in do_msgrcv(). Barrier (B).
  10355. - */
  10356. - smp_wmb();
  10357. + wake_q_add(wake_q, msr->r_tsk);
  10358. msr->r_msg = msg;
  10359. -
  10360. return 1;
  10361. }
  10362. }
  10363. @@ -613,6 +598,7 @@
  10364. struct msg_msg *msg;
  10365. int err;
  10366. struct ipc_namespace *ns;
  10367. + WAKE_Q(wake_q);
  10368. ns = current->nsproxy->ipc_ns;
  10369. @@ -698,7 +684,7 @@
  10370. msq->q_lspid = task_tgid_vnr(current);
  10371. msq->q_stime = get_seconds();
  10372. - if (!pipelined_send(msq, msg)) {
  10373. + if (!pipelined_send(msq, msg, &wake_q)) {
  10374. /* no one is waiting for this message, enqueue it */
  10375. list_add_tail(&msg->m_list, &msq->q_messages);
  10376. msq->q_cbytes += msgsz;
  10377. @@ -712,6 +698,7 @@
  10378. out_unlock0:
  10379. ipc_unlock_object(&msq->q_perm);
  10380. + wake_up_q(&wake_q);
  10381. out_unlock1:
  10382. rcu_read_unlock();
  10383. if (msg != NULL)
  10384. @@ -932,57 +919,25 @@
  10385. rcu_read_lock();
  10386. /* Lockless receive, part 2:
  10387. - * Wait until pipelined_send or expunge_all are outside of
  10388. - * wake_up_process(). There is a race with exit(), see
  10389. - * ipc/mqueue.c for the details. The correct serialization
  10390. - * ensures that a receiver cannot continue without the wakeup
  10391. - * being visibible _before_ setting r_msg:
  10392. + * The work in pipelined_send() and expunge_all():
  10393. + * - Set pointer to message
  10394. + * - Queue the receiver task for later wakeup
  10395. + * - Wake up the process after the lock is dropped.
  10396. *
  10397. - * CPU 0 CPU 1
  10398. - * <loop receiver>
  10399. - * smp_rmb(); (A) <-- pair -. <waker thread>
  10400. - * <load ->r_msg> | msr->r_msg = NULL;
  10401. - * | wake_up_process();
  10402. - * <continue> `------> smp_wmb(); (B)
  10403. - * msr->r_msg = msg;
  10404. - *
  10405. - * Where (A) orders the message value read and where (B) orders
  10406. - * the write to the r_msg -- done in both pipelined_send and
  10407. - * expunge_all.
  10408. + * Should the process wake up before this wakeup (due to a
  10409. + * signal) it will either see the message and continue …
  10410. */
  10411. - for (;;) {
  10412. - /*
  10413. - * Pairs with writer barrier in pipelined_send
  10414. - * or expunge_all.
  10415. - */
  10416. - smp_rmb(); /* barrier (A) */
  10417. - msg = (struct msg_msg *)msr_d.r_msg;
  10418. - if (msg)
  10419. - break;
  10420. - /*
  10421. - * The cpu_relax() call is a compiler barrier
  10422. - * which forces everything in this loop to be
  10423. - * re-loaded.
  10424. - */
  10425. - cpu_relax();
  10426. - }
  10427. -
  10428. - /* Lockless receive, part 3:
  10429. - * If there is a message or an error then accept it without
  10430. - * locking.
  10431. - */
  10432. + msg = (struct msg_msg *)msr_d.r_msg;
  10433. if (msg != ERR_PTR(-EAGAIN))
  10434. goto out_unlock1;
  10435. - /* Lockless receive, part 3:
  10436. - * Acquire the queue spinlock.
  10437. - */
  10438. + /*
  10439. + * … or see -EAGAIN, acquire the lock to check the message
  10440. + * again.
  10441. + */
  10442. ipc_lock_object(&msq->q_perm);
  10443. - /* Lockless receive, part 4:
  10444. - * Repeat test after acquiring the spinlock.
  10445. - */
  10446. msg = (struct msg_msg *)msr_d.r_msg;
  10447. if (msg != ERR_PTR(-EAGAIN))
  10448. goto out_unlock0;
  10449. diff -Nur linux-4.6.6.orig/ipc/sem.c linux-4.6.6/ipc/sem.c
  10450. --- linux-4.6.6.orig/ipc/sem.c 2016-08-10 12:54:57.000000000 +0200
  10451. +++ linux-4.6.6/ipc/sem.c 2016-08-22 21:27:47.357889524 +0200
  10452. @@ -697,6 +697,13 @@
  10453. static void wake_up_sem_queue_prepare(struct list_head *pt,
  10454. struct sem_queue *q, int error)
  10455. {
  10456. +#ifdef CONFIG_PREEMPT_RT_BASE
  10457. + struct task_struct *p = q->sleeper;
  10458. + get_task_struct(p);
  10459. + q->status = error;
  10460. + wake_up_process(p);
  10461. + put_task_struct(p);
  10462. +#else
  10463. if (list_empty(pt)) {
  10464. /*
  10465. * Hold preempt off so that we don't get preempted and have the
  10466. @@ -708,6 +715,7 @@
  10467. q->pid = error;
  10468. list_add_tail(&q->list, pt);
  10469. +#endif
  10470. }
  10471. /**
  10472. @@ -721,6 +729,7 @@
  10473. */
  10474. static void wake_up_sem_queue_do(struct list_head *pt)
  10475. {
  10476. +#ifndef CONFIG_PREEMPT_RT_BASE
  10477. struct sem_queue *q, *t;
  10478. int did_something;
  10479. @@ -733,6 +742,7 @@
  10480. }
  10481. if (did_something)
  10482. preempt_enable();
  10483. +#endif
  10484. }
  10485. static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
  10486. diff -Nur linux-4.6.6.orig/kernel/cgroup.c linux-4.6.6/kernel/cgroup.c
  10487. --- linux-4.6.6.orig/kernel/cgroup.c 2016-08-10 12:54:57.000000000 +0200
  10488. +++ linux-4.6.6/kernel/cgroup.c 2016-08-22 21:27:47.361889777 +0200
  10489. @@ -5011,10 +5011,10 @@
  10490. queue_work(cgroup_destroy_wq, &css->destroy_work);
  10491. }
  10492. -static void css_release_work_fn(struct work_struct *work)
  10493. +static void css_release_work_fn(struct swork_event *sev)
  10494. {
  10495. struct cgroup_subsys_state *css =
  10496. - container_of(work, struct cgroup_subsys_state, destroy_work);
  10497. + container_of(sev, struct cgroup_subsys_state, destroy_swork);
  10498. struct cgroup_subsys *ss = css->ss;
  10499. struct cgroup *cgrp = css->cgroup;
  10500. @@ -5055,8 +5055,8 @@
  10501. struct cgroup_subsys_state *css =
  10502. container_of(ref, struct cgroup_subsys_state, refcnt);
  10503. - INIT_WORK(&css->destroy_work, css_release_work_fn);
  10504. - queue_work(cgroup_destroy_wq, &css->destroy_work);
  10505. + INIT_SWORK(&css->destroy_swork, css_release_work_fn);
  10506. + swork_queue(&css->destroy_swork);
  10507. }
  10508. static void init_and_link_css(struct cgroup_subsys_state *css,
  10509. @@ -5698,6 +5698,7 @@
  10510. */
  10511. cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
  10512. BUG_ON(!cgroup_destroy_wq);
  10513. + BUG_ON(swork_get());
  10514. /*
  10515. * Used to destroy pidlists and separate to serve as flush domain.
  10516. diff -Nur linux-4.6.6.orig/kernel/cpu.c linux-4.6.6/kernel/cpu.c
  10517. --- linux-4.6.6.orig/kernel/cpu.c 2016-08-10 12:54:57.000000000 +0200
  10518. +++ linux-4.6.6/kernel/cpu.c 2016-08-22 21:27:47.361889777 +0200
  10519. @@ -152,8 +152,8 @@
  10520. #endif
  10521. } cpu_hotplug = {
  10522. .active_writer = NULL,
  10523. - .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
  10524. .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
  10525. + .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
  10526. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  10527. .dep_map = {.name = "cpu_hotplug.lock" },
  10528. #endif
  10529. @@ -166,6 +166,289 @@
  10530. #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
  10531. #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
  10532. +/**
  10533. + * hotplug_pcp - per cpu hotplug descriptor
  10534. + * @unplug: set when pin_current_cpu() needs to sync tasks
  10535. + * @sync_tsk: the task that waits for tasks to finish pinned sections
  10536. + * @refcount: counter of tasks in pinned sections
  10537. + * @grab_lock: set when the tasks entering pinned sections should wait
  10538. + * @synced: notifier for @sync_tsk to tell cpu_down it's finished
  10539. + * @mutex: the mutex to make tasks wait (used when @grab_lock is true)
  10540. + * @mutex_init: zero if the mutex hasn't been initialized yet.
  10541. + *
  10542. + * Although @unplug and @sync_tsk may point to the same task, the @unplug
  10543. + * is used as a flag and still exists after @sync_tsk has exited and
  10544. + * @sync_tsk set to NULL.
  10545. + */
  10546. +struct hotplug_pcp {
  10547. + struct task_struct *unplug;
  10548. + struct task_struct *sync_tsk;
  10549. + int refcount;
  10550. + int grab_lock;
  10551. + struct completion synced;
  10552. + struct completion unplug_wait;
  10553. +#ifdef CONFIG_PREEMPT_RT_FULL
  10554. + /*
  10555. + * Note, on PREEMPT_RT, the hotplug lock must save the state of
  10556. + * the task, otherwise the mutex will cause the task to fail
  10557. + * to sleep when required. (Because it's called from migrate_disable())
  10558. + *
  10559. + * The spinlock_t on PREEMPT_RT is a mutex that saves the task's
  10560. + * state.
  10561. + */
  10562. + spinlock_t lock;
  10563. +#else
  10564. + struct mutex mutex;
  10565. +#endif
  10566. + int mutex_init;
  10567. +};
  10568. +
  10569. +#ifdef CONFIG_PREEMPT_RT_FULL
  10570. +# define hotplug_lock(hp) rt_spin_lock__no_mg(&(hp)->lock)
  10571. +# define hotplug_unlock(hp) rt_spin_unlock__no_mg(&(hp)->lock)
  10572. +#else
  10573. +# define hotplug_lock(hp) mutex_lock(&(hp)->mutex)
  10574. +# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex)
  10575. +#endif
  10576. +
  10577. +static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp);
  10578. +
  10579. +/**
  10580. + * pin_current_cpu - Prevent the current cpu from being unplugged
  10581. + *
  10582. + * Lightweight version of get_online_cpus() to prevent cpu from being
  10583. + * unplugged when code runs in a migration disabled region.
  10584. + *
  10585. + * Must be called with preemption disabled (preempt_count = 1)!
  10586. + */
  10587. +void pin_current_cpu(void)
  10588. +{
  10589. + struct hotplug_pcp *hp;
  10590. + int force = 0;
  10591. +
  10592. +retry:
  10593. + hp = this_cpu_ptr(&hotplug_pcp);
  10594. +
  10595. + if (!hp->unplug || hp->refcount || force || preempt_count() > 1 ||
  10596. + hp->unplug == current) {
  10597. + hp->refcount++;
  10598. + return;
  10599. + }
  10600. + if (hp->grab_lock) {
  10601. + preempt_enable();
  10602. + hotplug_lock(hp);
  10603. + hotplug_unlock(hp);
  10604. + } else {
  10605. + preempt_enable();
  10606. + /*
  10607. + * Try to push this task off of this CPU.
  10608. + */
  10609. + if (!migrate_me()) {
  10610. + preempt_disable();
  10611. + hp = this_cpu_ptr(&hotplug_pcp);
  10612. + if (!hp->grab_lock) {
  10613. + /*
  10614. + * Just let it continue it's already pinned
  10615. + * or about to sleep.
  10616. + */
  10617. + force = 1;
  10618. + goto retry;
  10619. + }
  10620. + preempt_enable();
  10621. + }
  10622. + }
  10623. + preempt_disable();
  10624. + goto retry;
  10625. +}
  10626. +
  10627. +/**
  10628. + * unpin_current_cpu - Allow unplug of current cpu
  10629. + *
  10630. + * Must be called with preemption or interrupts disabled!
  10631. + */
  10632. +void unpin_current_cpu(void)
  10633. +{
  10634. + struct hotplug_pcp *hp = this_cpu_ptr(&hotplug_pcp);
  10635. +
  10636. + WARN_ON(hp->refcount <= 0);
  10637. +
  10638. + /* This is safe. sync_unplug_thread is pinned to this cpu */
  10639. + if (!--hp->refcount && hp->unplug && hp->unplug != current)
  10640. + wake_up_process(hp->unplug);
  10641. +}
  10642. +
  10643. +static void wait_for_pinned_cpus(struct hotplug_pcp *hp)
  10644. +{
  10645. + set_current_state(TASK_UNINTERRUPTIBLE);
  10646. + while (hp->refcount) {
  10647. + schedule_preempt_disabled();
  10648. + set_current_state(TASK_UNINTERRUPTIBLE);
  10649. + }
  10650. +}
  10651. +
  10652. +static int sync_unplug_thread(void *data)
  10653. +{
  10654. + struct hotplug_pcp *hp = data;
  10655. +
  10656. + wait_for_completion(&hp->unplug_wait);
  10657. + preempt_disable();
  10658. + hp->unplug = current;
  10659. + wait_for_pinned_cpus(hp);
  10660. +
  10661. + /*
  10662. + * This thread will synchronize the cpu_down() with threads
  10663. + * that have pinned the CPU. When the pinned CPU count reaches
  10664. + * zero, we inform the cpu_down code to continue to the next step.
  10665. + */
  10666. + set_current_state(TASK_UNINTERRUPTIBLE);
  10667. + preempt_enable();
  10668. + complete(&hp->synced);
  10669. +
  10670. + /*
  10671. + * If all succeeds, the next step will need tasks to wait till
  10672. + * the CPU is offline before continuing. To do this, the grab_lock
  10673. + * is set and tasks going into pin_current_cpu() will block on the
  10674. + * mutex. But we still need to wait for those that are already in
  10675. + * pinned CPU sections. If the cpu_down() failed, the kthread_should_stop()
  10676. + * will kick this thread out.
  10677. + */
  10678. + while (!hp->grab_lock && !kthread_should_stop()) {
  10679. + schedule();
  10680. + set_current_state(TASK_UNINTERRUPTIBLE);
  10681. + }
  10682. +
  10683. + /* Make sure grab_lock is seen before we see a stale completion */
  10684. + smp_mb();
  10685. +
  10686. + /*
  10687. + * Now just before cpu_down() enters stop machine, we need to make
  10688. + * sure all tasks that are in pinned CPU sections are out, and new
  10689. + * tasks will now grab the lock, keeping them from entering pinned
  10690. + * CPU sections.
  10691. + */
  10692. + if (!kthread_should_stop()) {
  10693. + preempt_disable();
  10694. + wait_for_pinned_cpus(hp);
  10695. + preempt_enable();
  10696. + complete(&hp->synced);
  10697. + }
  10698. +
  10699. + set_current_state(TASK_UNINTERRUPTIBLE);
  10700. + while (!kthread_should_stop()) {
  10701. + schedule();
  10702. + set_current_state(TASK_UNINTERRUPTIBLE);
  10703. + }
  10704. + set_current_state(TASK_RUNNING);
  10705. +
  10706. + /*
  10707. + * Force this thread off this CPU as it's going down and
  10708. + * we don't want any more work on this CPU.
  10709. + */
  10710. + current->flags &= ~PF_NO_SETAFFINITY;
  10711. + set_cpus_allowed_ptr(current, cpu_present_mask);
  10712. + migrate_me();
  10713. + return 0;
  10714. +}
  10715. +
  10716. +static void __cpu_unplug_sync(struct hotplug_pcp *hp)
  10717. +{
  10718. + wake_up_process(hp->sync_tsk);
  10719. + wait_for_completion(&hp->synced);
  10720. +}
  10721. +
  10722. +static void __cpu_unplug_wait(unsigned int cpu)
  10723. +{
  10724. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  10725. +
  10726. + complete(&hp->unplug_wait);
  10727. + wait_for_completion(&hp->synced);
  10728. +}
  10729. +
  10730. +/*
  10731. + * Start the sync_unplug_thread on the target cpu and wait for it to
  10732. + * complete.
  10733. + */
  10734. +static int cpu_unplug_begin(unsigned int cpu)
  10735. +{
  10736. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  10737. + int err;
  10738. +
  10739. + /* Protected by cpu_hotplug.lock */
  10740. + if (!hp->mutex_init) {
  10741. +#ifdef CONFIG_PREEMPT_RT_FULL
  10742. + spin_lock_init(&hp->lock);
  10743. +#else
  10744. + mutex_init(&hp->mutex);
  10745. +#endif
  10746. + hp->mutex_init = 1;
  10747. + }
  10748. +
  10749. + /* Inform the scheduler to migrate tasks off this CPU */
  10750. + tell_sched_cpu_down_begin(cpu);
  10751. +
  10752. + init_completion(&hp->synced);
  10753. + init_completion(&hp->unplug_wait);
  10754. +
  10755. + hp->sync_tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d", cpu);
  10756. + if (IS_ERR(hp->sync_tsk)) {
  10757. + err = PTR_ERR(hp->sync_tsk);
  10758. + hp->sync_tsk = NULL;
  10759. + return err;
  10760. + }
  10761. + kthread_bind(hp->sync_tsk, cpu);
  10762. +
  10763. + /*
  10764. + * Wait for tasks to get out of the pinned sections,
  10765. + * it's still OK if new tasks enter. Some CPU notifiers will
  10766. + * wait for tasks that are going to enter these sections and
  10767. + * we must not have them block.
  10768. + */
  10769. + wake_up_process(hp->sync_tsk);
  10770. + return 0;
  10771. +}
  10772. +
  10773. +static void cpu_unplug_sync(unsigned int cpu)
  10774. +{
  10775. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  10776. +
  10777. + init_completion(&hp->synced);
  10778. + /* The completion needs to be initialzied before setting grab_lock */
  10779. + smp_wmb();
  10780. +
  10781. + /* Grab the mutex before setting grab_lock */
  10782. + hotplug_lock(hp);
  10783. + hp->grab_lock = 1;
  10784. +
  10785. + /*
  10786. + * The CPU notifiers have been completed.
  10787. + * Wait for tasks to get out of pinned CPU sections and have new
  10788. + * tasks block until the CPU is completely down.
  10789. + */
  10790. + __cpu_unplug_sync(hp);
  10791. +
  10792. + /* All done with the sync thread */
  10793. + kthread_stop(hp->sync_tsk);
  10794. + hp->sync_tsk = NULL;
  10795. +}
  10796. +
  10797. +static void cpu_unplug_done(unsigned int cpu)
  10798. +{
  10799. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  10800. +
  10801. + hp->unplug = NULL;
  10802. + /* Let all tasks know cpu unplug is finished before cleaning up */
  10803. + smp_wmb();
  10804. +
  10805. + if (hp->sync_tsk)
  10806. + kthread_stop(hp->sync_tsk);
  10807. +
  10808. + if (hp->grab_lock) {
  10809. + hotplug_unlock(hp);
  10810. + /* protected by cpu_hotplug.lock */
  10811. + hp->grab_lock = 0;
  10812. + }
  10813. + tell_sched_cpu_down_done(cpu);
  10814. +}
  10815. void get_online_cpus(void)
  10816. {
  10817. @@ -718,10 +1001,14 @@
  10818. else
  10819. synchronize_rcu();
  10820. + __cpu_unplug_wait(cpu);
  10821. /* Park the smpboot threads */
  10822. kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
  10823. smpboot_park_threads(cpu);
  10824. + /* Notifiers are done. Don't let any more tasks pin this CPU. */
  10825. + cpu_unplug_sync(cpu);
  10826. +
  10827. /*
  10828. * Prevent irq alloc/free while the dying cpu reorganizes the
  10829. * interrupt affinities.
  10830. @@ -807,6 +1094,9 @@
  10831. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  10832. int prev_state, ret = 0;
  10833. bool hasdied = false;
  10834. + int mycpu;
  10835. + cpumask_var_t cpumask;
  10836. + cpumask_var_t cpumask_org;
  10837. if (num_online_cpus() == 1)
  10838. return -EBUSY;
  10839. @@ -814,7 +1104,34 @@
  10840. if (!cpu_present(cpu))
  10841. return -EINVAL;
  10842. + /* Move the downtaker off the unplug cpu */
  10843. + if (!alloc_cpumask_var(&cpumask, GFP_KERNEL))
  10844. + return -ENOMEM;
  10845. + if (!alloc_cpumask_var(&cpumask_org, GFP_KERNEL)) {
  10846. + free_cpumask_var(cpumask);
  10847. + return -ENOMEM;
  10848. + }
  10849. +
  10850. + cpumask_copy(cpumask_org, tsk_cpus_allowed(current));
  10851. + cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu));
  10852. + set_cpus_allowed_ptr(current, cpumask);
  10853. + free_cpumask_var(cpumask);
  10854. + migrate_disable();
  10855. + mycpu = smp_processor_id();
  10856. + if (mycpu == cpu) {
  10857. + printk(KERN_ERR "Yuck! Still on unplug CPU\n!");
  10858. + migrate_enable();
  10859. + ret = -EBUSY;
  10860. + goto restore_cpus;
  10861. + }
  10862. +
  10863. + migrate_enable();
  10864. cpu_hotplug_begin();
  10865. + ret = cpu_unplug_begin(cpu);
  10866. + if (ret) {
  10867. + printk("cpu_unplug_begin(%d) failed\n", cpu);
  10868. + goto out_cancel;
  10869. + }
  10870. cpuhp_tasks_frozen = tasks_frozen;
  10871. @@ -853,10 +1170,15 @@
  10872. hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE;
  10873. out:
  10874. + cpu_unplug_done(cpu);
  10875. +out_cancel:
  10876. cpu_hotplug_done();
  10877. /* This post dead nonsense must die */
  10878. if (!ret && hasdied)
  10879. cpu_notify_nofail(CPU_POST_DEAD, cpu);
  10880. +restore_cpus:
  10881. + set_cpus_allowed_ptr(current, cpumask_org);
  10882. + free_cpumask_var(cpumask_org);
  10883. return ret;
  10884. }
  10885. diff -Nur linux-4.6.6.orig/kernel/debug/kdb/kdb_io.c linux-4.6.6/kernel/debug/kdb/kdb_io.c
  10886. --- linux-4.6.6.orig/kernel/debug/kdb/kdb_io.c 2016-08-10 12:54:57.000000000 +0200
  10887. +++ linux-4.6.6/kernel/debug/kdb/kdb_io.c 2016-08-22 21:27:47.365890038 +0200
  10888. @@ -554,7 +554,6 @@
  10889. int linecount;
  10890. int colcount;
  10891. int logging, saved_loglevel = 0;
  10892. - int saved_trap_printk;
  10893. int got_printf_lock = 0;
  10894. int retlen = 0;
  10895. int fnd, len;
  10896. @@ -565,8 +564,6 @@
  10897. unsigned long uninitialized_var(flags);
  10898. preempt_disable();
  10899. - saved_trap_printk = kdb_trap_printk;
  10900. - kdb_trap_printk = 0;
  10901. /* Serialize kdb_printf if multiple cpus try to write at once.
  10902. * But if any cpu goes recursive in kdb, just print the output,
  10903. @@ -855,7 +852,6 @@
  10904. } else {
  10905. __release(kdb_printf_lock);
  10906. }
  10907. - kdb_trap_printk = saved_trap_printk;
  10908. preempt_enable();
  10909. return retlen;
  10910. }
  10911. @@ -865,9 +861,11 @@
  10912. va_list ap;
  10913. int r;
  10914. + kdb_trap_printk++;
  10915. va_start(ap, fmt);
  10916. r = vkdb_printf(KDB_MSGSRC_INTERNAL, fmt, ap);
  10917. va_end(ap);
  10918. + kdb_trap_printk--;
  10919. return r;
  10920. }
  10921. diff -Nur linux-4.6.6.orig/kernel/events/core.c linux-4.6.6/kernel/events/core.c
  10922. --- linux-4.6.6.orig/kernel/events/core.c 2016-08-10 12:54:57.000000000 +0200
  10923. +++ linux-4.6.6/kernel/events/core.c 2016-08-22 21:27:47.369890291 +0200
  10924. @@ -963,6 +963,7 @@
  10925. raw_spin_lock_init(&cpuctx->hrtimer_lock);
  10926. hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
  10927. timer->function = perf_mux_hrtimer_handler;
  10928. + timer->irqsafe = 1;
  10929. }
  10930. static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
  10931. @@ -7261,6 +7262,7 @@
  10932. hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  10933. hwc->hrtimer.function = perf_swevent_hrtimer;
  10934. + hwc->hrtimer.irqsafe = 1;
  10935. /*
  10936. * Since hrtimers have a fixed rate, we can do a static freq->period
  10937. diff -Nur linux-4.6.6.orig/kernel/exit.c linux-4.6.6/kernel/exit.c
  10938. --- linux-4.6.6.orig/kernel/exit.c 2016-08-10 12:54:57.000000000 +0200
  10939. +++ linux-4.6.6/kernel/exit.c 2016-08-22 21:27:47.369890291 +0200
  10940. @@ -143,7 +143,7 @@
  10941. * Do this under ->siglock, we can race with another thread
  10942. * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
  10943. */
  10944. - flush_sigqueue(&tsk->pending);
  10945. + flush_task_sigqueue(tsk);
  10946. tsk->sighand = NULL;
  10947. spin_unlock(&sighand->siglock);
  10948. diff -Nur linux-4.6.6.orig/kernel/fork.c linux-4.6.6/kernel/fork.c
  10949. --- linux-4.6.6.orig/kernel/fork.c 2016-08-10 12:54:57.000000000 +0200
  10950. +++ linux-4.6.6/kernel/fork.c 2016-08-22 21:27:47.373890551 +0200
  10951. @@ -253,7 +253,9 @@
  10952. if (atomic_dec_and_test(&sig->sigcnt))
  10953. free_signal_struct(sig);
  10954. }
  10955. -
  10956. +#ifdef CONFIG_PREEMPT_RT_BASE
  10957. +static
  10958. +#endif
  10959. void __put_task_struct(struct task_struct *tsk)
  10960. {
  10961. WARN_ON(!tsk->exit_state);
  10962. @@ -270,7 +272,18 @@
  10963. if (!profile_handoff_task(tsk))
  10964. free_task(tsk);
  10965. }
  10966. +#ifndef CONFIG_PREEMPT_RT_BASE
  10967. EXPORT_SYMBOL_GPL(__put_task_struct);
  10968. +#else
  10969. +void __put_task_struct_cb(struct rcu_head *rhp)
  10970. +{
  10971. + struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu);
  10972. +
  10973. + __put_task_struct(tsk);
  10974. +
  10975. +}
  10976. +EXPORT_SYMBOL_GPL(__put_task_struct_cb);
  10977. +#endif
  10978. void __init __weak arch_task_cache_init(void) { }
  10979. @@ -699,6 +712,19 @@
  10980. }
  10981. EXPORT_SYMBOL_GPL(__mmdrop);
  10982. +#ifdef CONFIG_PREEMPT_RT_BASE
  10983. +/*
  10984. + * RCU callback for delayed mm drop. Not strictly rcu, but we don't
  10985. + * want another facility to make this work.
  10986. + */
  10987. +void __mmdrop_delayed(struct rcu_head *rhp)
  10988. +{
  10989. + struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop);
  10990. +
  10991. + __mmdrop(mm);
  10992. +}
  10993. +#endif
  10994. +
  10995. /*
  10996. * Decrement the use count and release all resources for an mm.
  10997. */
  10998. @@ -1228,6 +1254,9 @@
  10999. */
  11000. static void posix_cpu_timers_init(struct task_struct *tsk)
  11001. {
  11002. +#ifdef CONFIG_PREEMPT_RT_BASE
  11003. + tsk->posix_timer_list = NULL;
  11004. +#endif
  11005. tsk->cputime_expires.prof_exp = 0;
  11006. tsk->cputime_expires.virt_exp = 0;
  11007. tsk->cputime_expires.sched_exp = 0;
  11008. @@ -1352,6 +1381,7 @@
  11009. spin_lock_init(&p->alloc_lock);
  11010. init_sigpending(&p->pending);
  11011. + p->sigqueue_cache = NULL;
  11012. p->utime = p->stime = p->gtime = 0;
  11013. p->utimescaled = p->stimescaled = 0;
  11014. diff -Nur linux-4.6.6.orig/kernel/futex.c linux-4.6.6/kernel/futex.c
  11015. --- linux-4.6.6.orig/kernel/futex.c 2016-08-10 12:54:57.000000000 +0200
  11016. +++ linux-4.6.6/kernel/futex.c 2016-08-22 21:27:47.373890551 +0200
  11017. @@ -874,7 +874,9 @@
  11018. * task still owns the PI-state:
  11019. */
  11020. if (head->next != next) {
  11021. + raw_spin_unlock_irq(&curr->pi_lock);
  11022. spin_unlock(&hb->lock);
  11023. + raw_spin_lock_irq(&curr->pi_lock);
  11024. continue;
  11025. }
  11026. @@ -1269,6 +1271,7 @@
  11027. struct futex_pi_state *pi_state = this->pi_state;
  11028. u32 uninitialized_var(curval), newval;
  11029. WAKE_Q(wake_q);
  11030. + WAKE_Q(wake_sleeper_q);
  11031. bool deboost;
  11032. int ret = 0;
  11033. @@ -1335,7 +1338,8 @@
  11034. raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  11035. - deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
  11036. + deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q,
  11037. + &wake_sleeper_q);
  11038. /*
  11039. * First unlock HB so the waiter does not spin on it once he got woken
  11040. @@ -1345,6 +1349,7 @@
  11041. */
  11042. spin_unlock(&hb->lock);
  11043. wake_up_q(&wake_q);
  11044. + wake_up_q_sleeper(&wake_sleeper_q);
  11045. if (deboost)
  11046. rt_mutex_adjust_prio(current);
  11047. @@ -1894,6 +1899,16 @@
  11048. requeue_pi_wake_futex(this, &key2, hb2);
  11049. drop_count++;
  11050. continue;
  11051. + } else if (ret == -EAGAIN) {
  11052. + /*
  11053. + * Waiter was woken by timeout or
  11054. + * signal and has set pi_blocked_on to
  11055. + * PI_WAKEUP_INPROGRESS before we
  11056. + * tried to enqueue it on the rtmutex.
  11057. + */
  11058. + this->pi_state = NULL;
  11059. + put_pi_state(pi_state);
  11060. + continue;
  11061. } else if (ret) {
  11062. /*
  11063. * rt_mutex_start_proxy_lock() detected a
  11064. @@ -2784,7 +2799,7 @@
  11065. struct hrtimer_sleeper timeout, *to = NULL;
  11066. struct rt_mutex_waiter rt_waiter;
  11067. struct rt_mutex *pi_mutex = NULL;
  11068. - struct futex_hash_bucket *hb;
  11069. + struct futex_hash_bucket *hb, *hb2;
  11070. union futex_key key2 = FUTEX_KEY_INIT;
  11071. struct futex_q q = futex_q_init;
  11072. int res, ret;
  11073. @@ -2809,10 +2824,7 @@
  11074. * The waiter is allocated on our stack, manipulated by the requeue
  11075. * code while we sleep on uaddr.
  11076. */
  11077. - debug_rt_mutex_init_waiter(&rt_waiter);
  11078. - RB_CLEAR_NODE(&rt_waiter.pi_tree_entry);
  11079. - RB_CLEAR_NODE(&rt_waiter.tree_entry);
  11080. - rt_waiter.task = NULL;
  11081. + rt_mutex_init_waiter(&rt_waiter, false);
  11082. ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
  11083. if (unlikely(ret != 0))
  11084. @@ -2843,20 +2855,55 @@
  11085. /* Queue the futex_q, drop the hb lock, wait for wakeup. */
  11086. futex_wait_queue_me(hb, &q, to);
  11087. - spin_lock(&hb->lock);
  11088. - ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
  11089. - spin_unlock(&hb->lock);
  11090. - if (ret)
  11091. - goto out_put_keys;
  11092. + /*
  11093. + * On RT we must avoid races with requeue and trying to block
  11094. + * on two mutexes (hb->lock and uaddr2's rtmutex) by
  11095. + * serializing access to pi_blocked_on with pi_lock.
  11096. + */
  11097. + raw_spin_lock_irq(&current->pi_lock);
  11098. + if (current->pi_blocked_on) {
  11099. + /*
  11100. + * We have been requeued or are in the process of
  11101. + * being requeued.
  11102. + */
  11103. + raw_spin_unlock_irq(&current->pi_lock);
  11104. + } else {
  11105. + /*
  11106. + * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS
  11107. + * prevents a concurrent requeue from moving us to the
  11108. + * uaddr2 rtmutex. After that we can safely acquire
  11109. + * (and possibly block on) hb->lock.
  11110. + */
  11111. + current->pi_blocked_on = PI_WAKEUP_INPROGRESS;
  11112. + raw_spin_unlock_irq(&current->pi_lock);
  11113. +
  11114. + spin_lock(&hb->lock);
  11115. +
  11116. + /*
  11117. + * Clean up pi_blocked_on. We might leak it otherwise
  11118. + * when we succeeded with the hb->lock in the fast
  11119. + * path.
  11120. + */
  11121. + raw_spin_lock_irq(&current->pi_lock);
  11122. + current->pi_blocked_on = NULL;
  11123. + raw_spin_unlock_irq(&current->pi_lock);
  11124. +
  11125. + ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
  11126. + spin_unlock(&hb->lock);
  11127. + if (ret)
  11128. + goto out_put_keys;
  11129. + }
  11130. /*
  11131. - * In order for us to be here, we know our q.key == key2, and since
  11132. - * we took the hb->lock above, we also know that futex_requeue() has
  11133. - * completed and we no longer have to concern ourselves with a wakeup
  11134. - * race with the atomic proxy lock acquisition by the requeue code. The
  11135. - * futex_requeue dropped our key1 reference and incremented our key2
  11136. - * reference count.
  11137. + * In order to be here, we have either been requeued, are in
  11138. + * the process of being requeued, or requeue successfully
  11139. + * acquired uaddr2 on our behalf. If pi_blocked_on was
  11140. + * non-null above, we may be racing with a requeue. Do not
  11141. + * rely on q->lock_ptr to be hb2->lock until after blocking on
  11142. + * hb->lock or hb2->lock. The futex_requeue dropped our key1
  11143. + * reference and incremented our key2 reference count.
  11144. */
  11145. + hb2 = hash_futex(&key2);
  11146. /* Check if the requeue code acquired the second futex for us. */
  11147. if (!q.rt_waiter) {
  11148. @@ -2865,14 +2912,15 @@
  11149. * did a lock-steal - fix up the PI-state in that case.
  11150. */
  11151. if (q.pi_state && (q.pi_state->owner != current)) {
  11152. - spin_lock(q.lock_ptr);
  11153. + spin_lock(&hb2->lock);
  11154. + BUG_ON(&hb2->lock != q.lock_ptr);
  11155. ret = fixup_pi_state_owner(uaddr2, &q, current);
  11156. /*
  11157. * Drop the reference to the pi state which
  11158. * the requeue_pi() code acquired for us.
  11159. */
  11160. put_pi_state(q.pi_state);
  11161. - spin_unlock(q.lock_ptr);
  11162. + spin_unlock(&hb2->lock);
  11163. }
  11164. } else {
  11165. /*
  11166. @@ -2885,7 +2933,8 @@
  11167. ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter);
  11168. debug_rt_mutex_free_waiter(&rt_waiter);
  11169. - spin_lock(q.lock_ptr);
  11170. + spin_lock(&hb2->lock);
  11171. + BUG_ON(&hb2->lock != q.lock_ptr);
  11172. /*
  11173. * Fixup the pi_state owner and possibly acquire the lock if we
  11174. * haven't already.
  11175. diff -Nur linux-4.6.6.orig/kernel/irq/handle.c linux-4.6.6/kernel/irq/handle.c
  11176. --- linux-4.6.6.orig/kernel/irq/handle.c 2016-08-10 12:54:57.000000000 +0200
  11177. +++ linux-4.6.6/kernel/irq/handle.c 2016-08-22 21:27:47.373890551 +0200
  11178. @@ -134,6 +134,8 @@
  11179. irqreturn_t handle_irq_event_percpu(struct irq_desc *desc)
  11180. {
  11181. + struct pt_regs *regs = get_irq_regs();
  11182. + u64 ip = regs ? instruction_pointer(regs) : 0;
  11183. irqreturn_t retval = IRQ_NONE;
  11184. unsigned int flags = 0, irq = desc->irq_data.irq;
  11185. struct irqaction *action;
  11186. @@ -174,7 +176,11 @@
  11187. retval |= res;
  11188. }
  11189. - add_interrupt_randomness(irq, flags);
  11190. +#ifdef CONFIG_PREEMPT_RT_FULL
  11191. + desc->random_ip = ip;
  11192. +#else
  11193. + add_interrupt_randomness(irq, flags, ip);
  11194. +#endif
  11195. if (!noirqdebug)
  11196. note_interrupt(desc, retval);
  11197. diff -Nur linux-4.6.6.orig/kernel/irq/manage.c linux-4.6.6/kernel/irq/manage.c
  11198. --- linux-4.6.6.orig/kernel/irq/manage.c 2016-08-10 12:54:57.000000000 +0200
  11199. +++ linux-4.6.6/kernel/irq/manage.c 2016-08-22 21:27:47.377890804 +0200
  11200. @@ -22,6 +22,7 @@
  11201. #include "internals.h"
  11202. #ifdef CONFIG_IRQ_FORCED_THREADING
  11203. +# ifndef CONFIG_PREEMPT_RT_BASE
  11204. __read_mostly bool force_irqthreads;
  11205. static int __init setup_forced_irqthreads(char *arg)
  11206. @@ -30,6 +31,7 @@
  11207. return 0;
  11208. }
  11209. early_param("threadirqs", setup_forced_irqthreads);
  11210. +# endif
  11211. #endif
  11212. static void __synchronize_hardirq(struct irq_desc *desc)
  11213. @@ -179,6 +181,62 @@
  11214. irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
  11215. #endif
  11216. +#ifdef CONFIG_PREEMPT_RT_FULL
  11217. +static void _irq_affinity_notify(struct irq_affinity_notify *notify);
  11218. +static struct task_struct *set_affinity_helper;
  11219. +static LIST_HEAD(affinity_list);
  11220. +static DEFINE_RAW_SPINLOCK(affinity_list_lock);
  11221. +
  11222. +static int set_affinity_thread(void *unused)
  11223. +{
  11224. + while (1) {
  11225. + struct irq_affinity_notify *notify;
  11226. + int empty;
  11227. +
  11228. + set_current_state(TASK_INTERRUPTIBLE);
  11229. +
  11230. + raw_spin_lock_irq(&affinity_list_lock);
  11231. + empty = list_empty(&affinity_list);
  11232. + raw_spin_unlock_irq(&affinity_list_lock);
  11233. +
  11234. + if (empty)
  11235. + schedule();
  11236. + if (kthread_should_stop())
  11237. + break;
  11238. + set_current_state(TASK_RUNNING);
  11239. +try_next:
  11240. + notify = NULL;
  11241. +
  11242. + raw_spin_lock_irq(&affinity_list_lock);
  11243. + if (!list_empty(&affinity_list)) {
  11244. + notify = list_first_entry(&affinity_list,
  11245. + struct irq_affinity_notify, list);
  11246. + list_del_init(&notify->list);
  11247. + }
  11248. + raw_spin_unlock_irq(&affinity_list_lock);
  11249. +
  11250. + if (!notify)
  11251. + continue;
  11252. + _irq_affinity_notify(notify);
  11253. + goto try_next;
  11254. + }
  11255. + return 0;
  11256. +}
  11257. +
  11258. +static void init_helper_thread(void)
  11259. +{
  11260. + if (set_affinity_helper)
  11261. + return;
  11262. + set_affinity_helper = kthread_run(set_affinity_thread, NULL,
  11263. + "affinity-cb");
  11264. + WARN_ON(IS_ERR(set_affinity_helper));
  11265. +}
  11266. +#else
  11267. +
  11268. +static inline void init_helper_thread(void) { }
  11269. +
  11270. +#endif
  11271. +
  11272. int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
  11273. bool force)
  11274. {
  11275. @@ -218,7 +276,17 @@
  11276. if (desc->affinity_notify) {
  11277. kref_get(&desc->affinity_notify->kref);
  11278. +
  11279. +#ifdef CONFIG_PREEMPT_RT_FULL
  11280. + raw_spin_lock(&affinity_list_lock);
  11281. + if (list_empty(&desc->affinity_notify->list))
  11282. + list_add_tail(&affinity_list,
  11283. + &desc->affinity_notify->list);
  11284. + raw_spin_unlock(&affinity_list_lock);
  11285. + wake_up_process(set_affinity_helper);
  11286. +#else
  11287. schedule_work(&desc->affinity_notify->work);
  11288. +#endif
  11289. }
  11290. irqd_set(data, IRQD_AFFINITY_SET);
  11291. @@ -256,10 +324,8 @@
  11292. }
  11293. EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
  11294. -static void irq_affinity_notify(struct work_struct *work)
  11295. +static void _irq_affinity_notify(struct irq_affinity_notify *notify)
  11296. {
  11297. - struct irq_affinity_notify *notify =
  11298. - container_of(work, struct irq_affinity_notify, work);
  11299. struct irq_desc *desc = irq_to_desc(notify->irq);
  11300. cpumask_var_t cpumask;
  11301. unsigned long flags;
  11302. @@ -281,6 +347,13 @@
  11303. kref_put(&notify->kref, notify->release);
  11304. }
  11305. +static void irq_affinity_notify(struct work_struct *work)
  11306. +{
  11307. + struct irq_affinity_notify *notify =
  11308. + container_of(work, struct irq_affinity_notify, work);
  11309. + _irq_affinity_notify(notify);
  11310. +}
  11311. +
  11312. /**
  11313. * irq_set_affinity_notifier - control notification of IRQ affinity changes
  11314. * @irq: Interrupt for which to enable/disable notification
  11315. @@ -310,6 +383,8 @@
  11316. notify->irq = irq;
  11317. kref_init(&notify->kref);
  11318. INIT_WORK(&notify->work, irq_affinity_notify);
  11319. + INIT_LIST_HEAD(&notify->list);
  11320. + init_helper_thread();
  11321. }
  11322. raw_spin_lock_irqsave(&desc->lock, flags);
  11323. @@ -863,7 +938,15 @@
  11324. local_bh_disable();
  11325. ret = action->thread_fn(action->irq, action->dev_id);
  11326. irq_finalize_oneshot(desc, action);
  11327. - local_bh_enable();
  11328. + /*
  11329. + * Interrupts which have real time requirements can be set up
  11330. + * to avoid softirq processing in the thread handler. This is
  11331. + * safe as these interrupts do not raise soft interrupts.
  11332. + */
  11333. + if (irq_settings_no_softirq_call(desc))
  11334. + _local_bh_enable();
  11335. + else
  11336. + local_bh_enable();
  11337. return ret;
  11338. }
  11339. @@ -960,6 +1043,12 @@
  11340. if (action_ret == IRQ_WAKE_THREAD)
  11341. irq_wake_secondary(desc, action);
  11342. +#ifdef CONFIG_PREEMPT_RT_FULL
  11343. + migrate_disable();
  11344. + add_interrupt_randomness(action->irq, 0,
  11345. + desc->random_ip ^ (unsigned long) action);
  11346. + migrate_enable();
  11347. +#endif
  11348. wake_threads_waitq(desc);
  11349. }
  11350. @@ -1313,6 +1402,9 @@
  11351. irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
  11352. }
  11353. + if (new->flags & IRQF_NO_SOFTIRQ_CALL)
  11354. + irq_settings_set_no_softirq_call(desc);
  11355. +
  11356. /* Set default affinity mask once everything is setup */
  11357. setup_affinity(desc, mask);
  11358. @@ -1998,7 +2090,7 @@
  11359. * This call sets the internal irqchip state of an interrupt,
  11360. * depending on the value of @which.
  11361. *
  11362. - * This function should be called with preemption disabled if the
  11363. + * This function should be called with migration disabled if the
  11364. * interrupt controller has per-cpu registers.
  11365. */
  11366. int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
  11367. diff -Nur linux-4.6.6.orig/kernel/irq/settings.h linux-4.6.6/kernel/irq/settings.h
  11368. --- linux-4.6.6.orig/kernel/irq/settings.h 2016-08-10 12:54:57.000000000 +0200
  11369. +++ linux-4.6.6/kernel/irq/settings.h 2016-08-22 21:27:47.377890804 +0200
  11370. @@ -16,6 +16,7 @@
  11371. _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID,
  11372. _IRQ_IS_POLLED = IRQ_IS_POLLED,
  11373. _IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY,
  11374. + _IRQ_NO_SOFTIRQ_CALL = IRQ_NO_SOFTIRQ_CALL,
  11375. _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK,
  11376. };
  11377. @@ -30,6 +31,7 @@
  11378. #define IRQ_PER_CPU_DEVID GOT_YOU_MORON
  11379. #define IRQ_IS_POLLED GOT_YOU_MORON
  11380. #define IRQ_DISABLE_UNLAZY GOT_YOU_MORON
  11381. +#define IRQ_NO_SOFTIRQ_CALL GOT_YOU_MORON
  11382. #undef IRQF_MODIFY_MASK
  11383. #define IRQF_MODIFY_MASK GOT_YOU_MORON
  11384. @@ -40,6 +42,16 @@
  11385. desc->status_use_accessors |= (set & _IRQF_MODIFY_MASK);
  11386. }
  11387. +static inline bool irq_settings_no_softirq_call(struct irq_desc *desc)
  11388. +{
  11389. + return desc->status_use_accessors & _IRQ_NO_SOFTIRQ_CALL;
  11390. +}
  11391. +
  11392. +static inline void irq_settings_set_no_softirq_call(struct irq_desc *desc)
  11393. +{
  11394. + desc->status_use_accessors |= _IRQ_NO_SOFTIRQ_CALL;
  11395. +}
  11396. +
  11397. static inline bool irq_settings_is_per_cpu(struct irq_desc *desc)
  11398. {
  11399. return desc->status_use_accessors & _IRQ_PER_CPU;
  11400. diff -Nur linux-4.6.6.orig/kernel/irq/spurious.c linux-4.6.6/kernel/irq/spurious.c
  11401. --- linux-4.6.6.orig/kernel/irq/spurious.c 2016-08-10 12:54:57.000000000 +0200
  11402. +++ linux-4.6.6/kernel/irq/spurious.c 2016-08-22 21:27:47.377890804 +0200
  11403. @@ -442,6 +442,10 @@
  11404. static int __init irqfixup_setup(char *str)
  11405. {
  11406. +#ifdef CONFIG_PREEMPT_RT_BASE
  11407. + pr_warn("irqfixup boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n");
  11408. + return 1;
  11409. +#endif
  11410. irqfixup = 1;
  11411. printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n");
  11412. printk(KERN_WARNING "This may impact system performance.\n");
  11413. @@ -454,6 +458,10 @@
  11414. static int __init irqpoll_setup(char *str)
  11415. {
  11416. +#ifdef CONFIG_PREEMPT_RT_BASE
  11417. + pr_warn("irqpoll boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n");
  11418. + return 1;
  11419. +#endif
  11420. irqfixup = 2;
  11421. printk(KERN_WARNING "Misrouted IRQ fixup and polling support "
  11422. "enabled\n");
  11423. diff -Nur linux-4.6.6.orig/kernel/irq_work.c linux-4.6.6/kernel/irq_work.c
  11424. --- linux-4.6.6.orig/kernel/irq_work.c 2016-08-10 12:54:57.000000000 +0200
  11425. +++ linux-4.6.6/kernel/irq_work.c 2016-08-22 21:27:47.377890804 +0200
  11426. @@ -17,6 +17,7 @@
  11427. #include <linux/cpu.h>
  11428. #include <linux/notifier.h>
  11429. #include <linux/smp.h>
  11430. +#include <linux/interrupt.h>
  11431. #include <asm/processor.h>
  11432. @@ -65,6 +66,8 @@
  11433. */
  11434. bool irq_work_queue_on(struct irq_work *work, int cpu)
  11435. {
  11436. + struct llist_head *list;
  11437. +
  11438. /* All work should have been flushed before going offline */
  11439. WARN_ON_ONCE(cpu_is_offline(cpu));
  11440. @@ -75,7 +78,12 @@
  11441. if (!irq_work_claim(work))
  11442. return false;
  11443. - if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
  11444. + if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ))
  11445. + list = &per_cpu(lazy_list, cpu);
  11446. + else
  11447. + list = &per_cpu(raised_list, cpu);
  11448. +
  11449. + if (llist_add(&work->llnode, list))
  11450. arch_send_call_function_single_ipi(cpu);
  11451. return true;
  11452. @@ -86,6 +94,9 @@
  11453. /* Enqueue the irq work @work on the current CPU */
  11454. bool irq_work_queue(struct irq_work *work)
  11455. {
  11456. + struct llist_head *list;
  11457. + bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL);
  11458. +
  11459. /* Only queue if not already pending */
  11460. if (!irq_work_claim(work))
  11461. return false;
  11462. @@ -93,13 +104,15 @@
  11463. /* Queue the entry and raise the IPI if needed. */
  11464. preempt_disable();
  11465. - /* If the work is "lazy", handle it from next tick if any */
  11466. - if (work->flags & IRQ_WORK_LAZY) {
  11467. - if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
  11468. - tick_nohz_tick_stopped())
  11469. - arch_irq_work_raise();
  11470. - } else {
  11471. - if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
  11472. + lazy_work = work->flags & IRQ_WORK_LAZY;
  11473. +
  11474. + if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ)))
  11475. + list = this_cpu_ptr(&lazy_list);
  11476. + else
  11477. + list = this_cpu_ptr(&raised_list);
  11478. +
  11479. + if (llist_add(&work->llnode, list)) {
  11480. + if (!lazy_work || tick_nohz_tick_stopped())
  11481. arch_irq_work_raise();
  11482. }
  11483. @@ -116,9 +129,8 @@
  11484. raised = this_cpu_ptr(&raised_list);
  11485. lazy = this_cpu_ptr(&lazy_list);
  11486. - if (llist_empty(raised) || arch_irq_work_has_interrupt())
  11487. - if (llist_empty(lazy))
  11488. - return false;
  11489. + if (llist_empty(raised) && llist_empty(lazy))
  11490. + return false;
  11491. /* All work should have been flushed before going offline */
  11492. WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
  11493. @@ -132,7 +144,7 @@
  11494. struct irq_work *work;
  11495. struct llist_node *llnode;
  11496. - BUG_ON(!irqs_disabled());
  11497. + BUG_ON_NONRT(!irqs_disabled());
  11498. if (llist_empty(list))
  11499. return;
  11500. @@ -169,7 +181,16 @@
  11501. void irq_work_run(void)
  11502. {
  11503. irq_work_run_list(this_cpu_ptr(&raised_list));
  11504. - irq_work_run_list(this_cpu_ptr(&lazy_list));
  11505. + if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) {
  11506. + /*
  11507. + * NOTE: we raise softirq via IPI for safety,
  11508. + * and execute in irq_work_tick() to move the
  11509. + * overhead from hard to soft irq context.
  11510. + */
  11511. + if (!llist_empty(this_cpu_ptr(&lazy_list)))
  11512. + raise_softirq(TIMER_SOFTIRQ);
  11513. + } else
  11514. + irq_work_run_list(this_cpu_ptr(&lazy_list));
  11515. }
  11516. EXPORT_SYMBOL_GPL(irq_work_run);
  11517. @@ -179,8 +200,17 @@
  11518. if (!llist_empty(raised) && !arch_irq_work_has_interrupt())
  11519. irq_work_run_list(raised);
  11520. +
  11521. + if (!IS_ENABLED(CONFIG_PREEMPT_RT_FULL))
  11522. + irq_work_run_list(this_cpu_ptr(&lazy_list));
  11523. +}
  11524. +
  11525. +#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL)
  11526. +void irq_work_tick_soft(void)
  11527. +{
  11528. irq_work_run_list(this_cpu_ptr(&lazy_list));
  11529. }
  11530. +#endif
  11531. /*
  11532. * Synchronize against the irq_work @entry, ensures the entry is not
  11533. diff -Nur linux-4.6.6.orig/kernel/Kconfig.locks linux-4.6.6/kernel/Kconfig.locks
  11534. --- linux-4.6.6.orig/kernel/Kconfig.locks 2016-08-10 12:54:57.000000000 +0200
  11535. +++ linux-4.6.6/kernel/Kconfig.locks 2016-08-22 21:27:47.357889524 +0200
  11536. @@ -225,11 +225,11 @@
  11537. config MUTEX_SPIN_ON_OWNER
  11538. def_bool y
  11539. - depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW
  11540. + depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL
  11541. config RWSEM_SPIN_ON_OWNER
  11542. def_bool y
  11543. - depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW
  11544. + depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL
  11545. config LOCK_SPIN_ON_OWNER
  11546. def_bool y
  11547. diff -Nur linux-4.6.6.orig/kernel/Kconfig.preempt linux-4.6.6/kernel/Kconfig.preempt
  11548. --- linux-4.6.6.orig/kernel/Kconfig.preempt 2016-08-10 12:54:57.000000000 +0200
  11549. +++ linux-4.6.6/kernel/Kconfig.preempt 2016-08-22 21:27:47.357889524 +0200
  11550. @@ -1,3 +1,16 @@
  11551. +config PREEMPT
  11552. + bool
  11553. + select PREEMPT_COUNT
  11554. +
  11555. +config PREEMPT_RT_BASE
  11556. + bool
  11557. + select PREEMPT
  11558. +
  11559. +config HAVE_PREEMPT_LAZY
  11560. + bool
  11561. +
  11562. +config PREEMPT_LAZY
  11563. + def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL
  11564. choice
  11565. prompt "Preemption Model"
  11566. @@ -33,9 +46,9 @@
  11567. Select this if you are building a kernel for a desktop system.
  11568. -config PREEMPT
  11569. +config PREEMPT__LL
  11570. bool "Preemptible Kernel (Low-Latency Desktop)"
  11571. - select PREEMPT_COUNT
  11572. + select PREEMPT
  11573. select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
  11574. help
  11575. This option reduces the latency of the kernel by making
  11576. @@ -52,6 +65,22 @@
  11577. embedded system with latency requirements in the milliseconds
  11578. range.
  11579. +config PREEMPT_RTB
  11580. + bool "Preemptible Kernel (Basic RT)"
  11581. + select PREEMPT_RT_BASE
  11582. + help
  11583. + This option is basically the same as (Low-Latency Desktop) but
  11584. + enables changes which are preliminary for the full preemptible
  11585. + RT kernel.
  11586. +
  11587. +config PREEMPT_RT_FULL
  11588. + bool "Fully Preemptible Kernel (RT)"
  11589. + depends on IRQ_FORCED_THREADING
  11590. + select PREEMPT_RT_BASE
  11591. + select PREEMPT_RCU
  11592. + help
  11593. + All and everything
  11594. +
  11595. endchoice
  11596. config PREEMPT_COUNT
  11597. diff -Nur linux-4.6.6.orig/kernel/ksysfs.c linux-4.6.6/kernel/ksysfs.c
  11598. --- linux-4.6.6.orig/kernel/ksysfs.c 2016-08-10 12:54:57.000000000 +0200
  11599. +++ linux-4.6.6/kernel/ksysfs.c 2016-08-22 21:27:47.377890804 +0200
  11600. @@ -136,6 +136,15 @@
  11601. #endif /* CONFIG_KEXEC_CORE */
  11602. +#if defined(CONFIG_PREEMPT_RT_FULL)
  11603. +static ssize_t realtime_show(struct kobject *kobj,
  11604. + struct kobj_attribute *attr, char *buf)
  11605. +{
  11606. + return sprintf(buf, "%d\n", 1);
  11607. +}
  11608. +KERNEL_ATTR_RO(realtime);
  11609. +#endif
  11610. +
  11611. /* whether file capabilities are enabled */
  11612. static ssize_t fscaps_show(struct kobject *kobj,
  11613. struct kobj_attribute *attr, char *buf)
  11614. @@ -225,6 +234,9 @@
  11615. &rcu_expedited_attr.attr,
  11616. &rcu_normal_attr.attr,
  11617. #endif
  11618. +#ifdef CONFIG_PREEMPT_RT_FULL
  11619. + &realtime_attr.attr,
  11620. +#endif
  11621. NULL
  11622. };
  11623. diff -Nur linux-4.6.6.orig/kernel/locking/lglock.c linux-4.6.6/kernel/locking/lglock.c
  11624. --- linux-4.6.6.orig/kernel/locking/lglock.c 2016-08-10 12:54:57.000000000 +0200
  11625. +++ linux-4.6.6/kernel/locking/lglock.c 2016-08-22 21:27:47.377890804 +0200
  11626. @@ -4,6 +4,15 @@
  11627. #include <linux/cpu.h>
  11628. #include <linux/string.h>
  11629. +#ifndef CONFIG_PREEMPT_RT_FULL
  11630. +# define lg_lock_ptr arch_spinlock_t
  11631. +# define lg_do_lock(l) arch_spin_lock(l)
  11632. +# define lg_do_unlock(l) arch_spin_unlock(l)
  11633. +#else
  11634. +# define lg_lock_ptr struct rt_mutex
  11635. +# define lg_do_lock(l) __rt_spin_lock__no_mg(l)
  11636. +# define lg_do_unlock(l) __rt_spin_unlock(l)
  11637. +#endif
  11638. /*
  11639. * Note there is no uninit, so lglocks cannot be defined in
  11640. * modules (but it's fine to use them from there)
  11641. @@ -12,51 +21,60 @@
  11642. void lg_lock_init(struct lglock *lg, char *name)
  11643. {
  11644. +#ifdef CONFIG_PREEMPT_RT_FULL
  11645. + int i;
  11646. +
  11647. + for_each_possible_cpu(i) {
  11648. + struct rt_mutex *lock = per_cpu_ptr(lg->lock, i);
  11649. +
  11650. + rt_mutex_init(lock);
  11651. + }
  11652. +#endif
  11653. LOCKDEP_INIT_MAP(&lg->lock_dep_map, name, &lg->lock_key, 0);
  11654. }
  11655. EXPORT_SYMBOL(lg_lock_init);
  11656. void lg_local_lock(struct lglock *lg)
  11657. {
  11658. - arch_spinlock_t *lock;
  11659. + lg_lock_ptr *lock;
  11660. - preempt_disable();
  11661. + migrate_disable();
  11662. lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
  11663. lock = this_cpu_ptr(lg->lock);
  11664. - arch_spin_lock(lock);
  11665. + lg_do_lock(lock);
  11666. }
  11667. EXPORT_SYMBOL(lg_local_lock);
  11668. void lg_local_unlock(struct lglock *lg)
  11669. {
  11670. - arch_spinlock_t *lock;
  11671. + lg_lock_ptr *lock;
  11672. lock_release(&lg->lock_dep_map, 1, _RET_IP_);
  11673. lock = this_cpu_ptr(lg->lock);
  11674. - arch_spin_unlock(lock);
  11675. - preempt_enable();
  11676. + lg_do_unlock(lock);
  11677. + migrate_enable();
  11678. }
  11679. EXPORT_SYMBOL(lg_local_unlock);
  11680. void lg_local_lock_cpu(struct lglock *lg, int cpu)
  11681. {
  11682. - arch_spinlock_t *lock;
  11683. + lg_lock_ptr *lock;
  11684. - preempt_disable();
  11685. + preempt_disable_nort();
  11686. lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
  11687. lock = per_cpu_ptr(lg->lock, cpu);
  11688. - arch_spin_lock(lock);
  11689. + lg_do_lock(lock);
  11690. }
  11691. EXPORT_SYMBOL(lg_local_lock_cpu);
  11692. void lg_local_unlock_cpu(struct lglock *lg, int cpu)
  11693. {
  11694. - arch_spinlock_t *lock;
  11695. + lg_lock_ptr *lock;
  11696. lock_release(&lg->lock_dep_map, 1, _RET_IP_);
  11697. lock = per_cpu_ptr(lg->lock, cpu);
  11698. - arch_spin_unlock(lock);
  11699. - preempt_enable();
  11700. + lg_do_unlock(lock);
  11701. + preempt_enable_nort();
  11702. }
  11703. EXPORT_SYMBOL(lg_local_unlock_cpu);
  11704. @@ -68,30 +86,30 @@
  11705. if (cpu2 < cpu1)
  11706. swap(cpu1, cpu2);
  11707. - preempt_disable();
  11708. + preempt_disable_nort();
  11709. lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
  11710. - arch_spin_lock(per_cpu_ptr(lg->lock, cpu1));
  11711. - arch_spin_lock(per_cpu_ptr(lg->lock, cpu2));
  11712. + lg_do_lock(per_cpu_ptr(lg->lock, cpu1));
  11713. + lg_do_lock(per_cpu_ptr(lg->lock, cpu2));
  11714. }
  11715. void lg_double_unlock(struct lglock *lg, int cpu1, int cpu2)
  11716. {
  11717. lock_release(&lg->lock_dep_map, 1, _RET_IP_);
  11718. - arch_spin_unlock(per_cpu_ptr(lg->lock, cpu1));
  11719. - arch_spin_unlock(per_cpu_ptr(lg->lock, cpu2));
  11720. - preempt_enable();
  11721. + lg_do_unlock(per_cpu_ptr(lg->lock, cpu1));
  11722. + lg_do_unlock(per_cpu_ptr(lg->lock, cpu2));
  11723. + preempt_enable_nort();
  11724. }
  11725. void lg_global_lock(struct lglock *lg)
  11726. {
  11727. int i;
  11728. - preempt_disable();
  11729. + preempt_disable_nort();
  11730. lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
  11731. for_each_possible_cpu(i) {
  11732. - arch_spinlock_t *lock;
  11733. + lg_lock_ptr *lock;
  11734. lock = per_cpu_ptr(lg->lock, i);
  11735. - arch_spin_lock(lock);
  11736. + lg_do_lock(lock);
  11737. }
  11738. }
  11739. EXPORT_SYMBOL(lg_global_lock);
  11740. @@ -102,10 +120,35 @@
  11741. lock_release(&lg->lock_dep_map, 1, _RET_IP_);
  11742. for_each_possible_cpu(i) {
  11743. - arch_spinlock_t *lock;
  11744. + lg_lock_ptr *lock;
  11745. lock = per_cpu_ptr(lg->lock, i);
  11746. - arch_spin_unlock(lock);
  11747. + lg_do_unlock(lock);
  11748. }
  11749. - preempt_enable();
  11750. + preempt_enable_nort();
  11751. }
  11752. EXPORT_SYMBOL(lg_global_unlock);
  11753. +
  11754. +#ifdef CONFIG_PREEMPT_RT_FULL
  11755. +/*
  11756. + * HACK: If you use this, you get to keep the pieces.
  11757. + * Used in queue_stop_cpus_work() when stop machinery
  11758. + * is called from inactive CPU, so we can't schedule.
  11759. + */
  11760. +# define lg_do_trylock_relax(l) \
  11761. + do { \
  11762. + while (!__rt_spin_trylock(l)) \
  11763. + cpu_relax(); \
  11764. + } while (0)
  11765. +
  11766. +void lg_global_trylock_relax(struct lglock *lg)
  11767. +{
  11768. + int i;
  11769. +
  11770. + lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
  11771. + for_each_possible_cpu(i) {
  11772. + lg_lock_ptr *lock;
  11773. + lock = per_cpu_ptr(lg->lock, i);
  11774. + lg_do_trylock_relax(lock);
  11775. + }
  11776. +}
  11777. +#endif
  11778. diff -Nur linux-4.6.6.orig/kernel/locking/lockdep.c linux-4.6.6/kernel/locking/lockdep.c
  11779. --- linux-4.6.6.orig/kernel/locking/lockdep.c 2016-08-10 12:54:57.000000000 +0200
  11780. +++ linux-4.6.6/kernel/locking/lockdep.c 2016-08-22 21:27:47.381891064 +0200
  11781. @@ -3648,6 +3648,7 @@
  11782. }
  11783. }
  11784. +#ifndef CONFIG_PREEMPT_RT_FULL
  11785. /*
  11786. * We dont accurately track softirq state in e.g.
  11787. * hardirq contexts (such as on 4KSTACKS), so only
  11788. @@ -3662,6 +3663,7 @@
  11789. DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
  11790. }
  11791. }
  11792. +#endif
  11793. if (!debug_locks)
  11794. print_irqtrace_events(current);
  11795. diff -Nur linux-4.6.6.orig/kernel/locking/locktorture.c linux-4.6.6/kernel/locking/locktorture.c
  11796. --- linux-4.6.6.orig/kernel/locking/locktorture.c 2016-08-10 12:54:57.000000000 +0200
  11797. +++ linux-4.6.6/kernel/locking/locktorture.c 2016-08-22 21:27:47.381891064 +0200
  11798. @@ -26,7 +26,6 @@
  11799. #include <linux/kthread.h>
  11800. #include <linux/sched/rt.h>
  11801. #include <linux/spinlock.h>
  11802. -#include <linux/rwlock.h>
  11803. #include <linux/mutex.h>
  11804. #include <linux/rwsem.h>
  11805. #include <linux/smp.h>
  11806. diff -Nur linux-4.6.6.orig/kernel/locking/Makefile linux-4.6.6/kernel/locking/Makefile
  11807. --- linux-4.6.6.orig/kernel/locking/Makefile 2016-08-10 12:54:57.000000000 +0200
  11808. +++ linux-4.6.6/kernel/locking/Makefile 2016-08-22 21:27:47.377890804 +0200
  11809. @@ -2,7 +2,7 @@
  11810. # and is generally not a function of system call inputs.
  11811. KCOV_INSTRUMENT := n
  11812. -obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
  11813. +obj-y += semaphore.o percpu-rwsem.o
  11814. ifdef CONFIG_FUNCTION_TRACER
  11815. CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE)
  11816. @@ -11,7 +11,11 @@
  11817. CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE)
  11818. endif
  11819. +ifneq ($(CONFIG_PREEMPT_RT_FULL),y)
  11820. +obj-y += mutex.o
  11821. obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o
  11822. +obj-y += rwsem.o
  11823. +endif
  11824. obj-$(CONFIG_LOCKDEP) += lockdep.o
  11825. ifeq ($(CONFIG_PROC_FS),y)
  11826. obj-$(CONFIG_LOCKDEP) += lockdep_proc.o
  11827. @@ -25,7 +29,10 @@
  11828. obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
  11829. obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
  11830. obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
  11831. +ifneq ($(CONFIG_PREEMPT_RT_FULL),y)
  11832. obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o
  11833. obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o
  11834. +endif
  11835. +obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o
  11836. obj-$(CONFIG_QUEUED_RWLOCKS) += qrwlock.o
  11837. obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o
  11838. diff -Nur linux-4.6.6.orig/kernel/locking/rt.c linux-4.6.6/kernel/locking/rt.c
  11839. --- linux-4.6.6.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100
  11840. +++ linux-4.6.6/kernel/locking/rt.c 2016-08-22 21:27:47.385891317 +0200
  11841. @@ -0,0 +1,474 @@
  11842. +/*
  11843. + * kernel/rt.c
  11844. + *
  11845. + * Real-Time Preemption Support
  11846. + *
  11847. + * started by Ingo Molnar:
  11848. + *
  11849. + * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  11850. + * Copyright (C) 2006, Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  11851. + *
  11852. + * historic credit for proving that Linux spinlocks can be implemented via
  11853. + * RT-aware mutexes goes to many people: The Pmutex project (Dirk Grambow
  11854. + * and others) who prototyped it on 2.4 and did lots of comparative
  11855. + * research and analysis; TimeSys, for proving that you can implement a
  11856. + * fully preemptible kernel via the use of IRQ threading and mutexes;
  11857. + * Bill Huey for persuasively arguing on lkml that the mutex model is the
  11858. + * right one; and to MontaVista, who ported pmutexes to 2.6.
  11859. + *
  11860. + * This code is a from-scratch implementation and is not based on pmutexes,
  11861. + * but the idea of converting spinlocks to mutexes is used here too.
  11862. + *
  11863. + * lock debugging, locking tree, deadlock detection:
  11864. + *
  11865. + * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey
  11866. + * Released under the General Public License (GPL).
  11867. + *
  11868. + * Includes portions of the generic R/W semaphore implementation from:
  11869. + *
  11870. + * Copyright (c) 2001 David Howells (dhowells@redhat.com).
  11871. + * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de>
  11872. + * - Derived also from comments by Linus
  11873. + *
  11874. + * Pending ownership of locks and ownership stealing:
  11875. + *
  11876. + * Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt
  11877. + *
  11878. + * (also by Steven Rostedt)
  11879. + * - Converted single pi_lock to individual task locks.
  11880. + *
  11881. + * By Esben Nielsen:
  11882. + * Doing priority inheritance with help of the scheduler.
  11883. + *
  11884. + * Copyright (C) 2006, Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  11885. + * - major rework based on Esben Nielsens initial patch
  11886. + * - replaced thread_info references by task_struct refs
  11887. + * - removed task->pending_owner dependency
  11888. + * - BKL drop/reacquire for semaphore style locks to avoid deadlocks
  11889. + * in the scheduler return path as discussed with Steven Rostedt
  11890. + *
  11891. + * Copyright (C) 2006, Kihon Technologies Inc.
  11892. + * Steven Rostedt <rostedt@goodmis.org>
  11893. + * - debugged and patched Thomas Gleixner's rework.
  11894. + * - added back the cmpxchg to the rework.
  11895. + * - turned atomic require back on for SMP.
  11896. + */
  11897. +
  11898. +#include <linux/spinlock.h>
  11899. +#include <linux/rtmutex.h>
  11900. +#include <linux/sched.h>
  11901. +#include <linux/delay.h>
  11902. +#include <linux/module.h>
  11903. +#include <linux/kallsyms.h>
  11904. +#include <linux/syscalls.h>
  11905. +#include <linux/interrupt.h>
  11906. +#include <linux/plist.h>
  11907. +#include <linux/fs.h>
  11908. +#include <linux/futex.h>
  11909. +#include <linux/hrtimer.h>
  11910. +
  11911. +#include "rtmutex_common.h"
  11912. +
  11913. +/*
  11914. + * struct mutex functions
  11915. + */
  11916. +void __mutex_do_init(struct mutex *mutex, const char *name,
  11917. + struct lock_class_key *key)
  11918. +{
  11919. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  11920. + /*
  11921. + * Make sure we are not reinitializing a held lock:
  11922. + */
  11923. + debug_check_no_locks_freed((void *)mutex, sizeof(*mutex));
  11924. + lockdep_init_map(&mutex->dep_map, name, key, 0);
  11925. +#endif
  11926. + mutex->lock.save_state = 0;
  11927. +}
  11928. +EXPORT_SYMBOL(__mutex_do_init);
  11929. +
  11930. +void __lockfunc _mutex_lock(struct mutex *lock)
  11931. +{
  11932. + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  11933. + rt_mutex_lock(&lock->lock);
  11934. +}
  11935. +EXPORT_SYMBOL(_mutex_lock);
  11936. +
  11937. +int __lockfunc _mutex_lock_interruptible(struct mutex *lock)
  11938. +{
  11939. + int ret;
  11940. +
  11941. + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  11942. + ret = rt_mutex_lock_interruptible(&lock->lock);
  11943. + if (ret)
  11944. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  11945. + return ret;
  11946. +}
  11947. +EXPORT_SYMBOL(_mutex_lock_interruptible);
  11948. +
  11949. +int __lockfunc _mutex_lock_killable(struct mutex *lock)
  11950. +{
  11951. + int ret;
  11952. +
  11953. + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  11954. + ret = rt_mutex_lock_killable(&lock->lock);
  11955. + if (ret)
  11956. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  11957. + return ret;
  11958. +}
  11959. +EXPORT_SYMBOL(_mutex_lock_killable);
  11960. +
  11961. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  11962. +void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass)
  11963. +{
  11964. + mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_);
  11965. + rt_mutex_lock(&lock->lock);
  11966. +}
  11967. +EXPORT_SYMBOL(_mutex_lock_nested);
  11968. +
  11969. +void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
  11970. +{
  11971. + mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_);
  11972. + rt_mutex_lock(&lock->lock);
  11973. +}
  11974. +EXPORT_SYMBOL(_mutex_lock_nest_lock);
  11975. +
  11976. +int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass)
  11977. +{
  11978. + int ret;
  11979. +
  11980. + mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_);
  11981. + ret = rt_mutex_lock_interruptible(&lock->lock);
  11982. + if (ret)
  11983. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  11984. + return ret;
  11985. +}
  11986. +EXPORT_SYMBOL(_mutex_lock_interruptible_nested);
  11987. +
  11988. +int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass)
  11989. +{
  11990. + int ret;
  11991. +
  11992. + mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  11993. + ret = rt_mutex_lock_killable(&lock->lock);
  11994. + if (ret)
  11995. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  11996. + return ret;
  11997. +}
  11998. +EXPORT_SYMBOL(_mutex_lock_killable_nested);
  11999. +#endif
  12000. +
  12001. +int __lockfunc _mutex_trylock(struct mutex *lock)
  12002. +{
  12003. + int ret = rt_mutex_trylock(&lock->lock);
  12004. +
  12005. + if (ret)
  12006. + mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12007. +
  12008. + return ret;
  12009. +}
  12010. +EXPORT_SYMBOL(_mutex_trylock);
  12011. +
  12012. +void __lockfunc _mutex_unlock(struct mutex *lock)
  12013. +{
  12014. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  12015. + rt_mutex_unlock(&lock->lock);
  12016. +}
  12017. +EXPORT_SYMBOL(_mutex_unlock);
  12018. +
  12019. +/*
  12020. + * rwlock_t functions
  12021. + */
  12022. +int __lockfunc rt_write_trylock(rwlock_t *rwlock)
  12023. +{
  12024. + int ret;
  12025. +
  12026. + migrate_disable();
  12027. + ret = rt_mutex_trylock(&rwlock->lock);
  12028. + if (ret)
  12029. + rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
  12030. + else
  12031. + migrate_enable();
  12032. +
  12033. + return ret;
  12034. +}
  12035. +EXPORT_SYMBOL(rt_write_trylock);
  12036. +
  12037. +int __lockfunc rt_write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags)
  12038. +{
  12039. + int ret;
  12040. +
  12041. + *flags = 0;
  12042. + ret = rt_write_trylock(rwlock);
  12043. + return ret;
  12044. +}
  12045. +EXPORT_SYMBOL(rt_write_trylock_irqsave);
  12046. +
  12047. +int __lockfunc rt_read_trylock(rwlock_t *rwlock)
  12048. +{
  12049. + struct rt_mutex *lock = &rwlock->lock;
  12050. + int ret = 1;
  12051. +
  12052. + /*
  12053. + * recursive read locks succeed when current owns the lock,
  12054. + * but not when read_depth == 0 which means that the lock is
  12055. + * write locked.
  12056. + */
  12057. + if (rt_mutex_owner(lock) != current) {
  12058. + migrate_disable();
  12059. + ret = rt_mutex_trylock(lock);
  12060. + if (ret)
  12061. + rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
  12062. + else
  12063. + migrate_enable();
  12064. +
  12065. + } else if (!rwlock->read_depth) {
  12066. + ret = 0;
  12067. + }
  12068. +
  12069. + if (ret)
  12070. + rwlock->read_depth++;
  12071. +
  12072. + return ret;
  12073. +}
  12074. +EXPORT_SYMBOL(rt_read_trylock);
  12075. +
  12076. +void __lockfunc rt_write_lock(rwlock_t *rwlock)
  12077. +{
  12078. + rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
  12079. + __rt_spin_lock(&rwlock->lock);
  12080. +}
  12081. +EXPORT_SYMBOL(rt_write_lock);
  12082. +
  12083. +void __lockfunc rt_read_lock(rwlock_t *rwlock)
  12084. +{
  12085. + struct rt_mutex *lock = &rwlock->lock;
  12086. +
  12087. +
  12088. + /*
  12089. + * recursive read locks succeed when current owns the lock
  12090. + */
  12091. + if (rt_mutex_owner(lock) != current) {
  12092. + rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
  12093. + __rt_spin_lock(lock);
  12094. + }
  12095. + rwlock->read_depth++;
  12096. +}
  12097. +
  12098. +EXPORT_SYMBOL(rt_read_lock);
  12099. +
  12100. +void __lockfunc rt_write_unlock(rwlock_t *rwlock)
  12101. +{
  12102. + /* NOTE: we always pass in '1' for nested, for simplicity */
  12103. + rwlock_release(&rwlock->dep_map, 1, _RET_IP_);
  12104. + __rt_spin_unlock(&rwlock->lock);
  12105. + migrate_enable();
  12106. +}
  12107. +EXPORT_SYMBOL(rt_write_unlock);
  12108. +
  12109. +void __lockfunc rt_read_unlock(rwlock_t *rwlock)
  12110. +{
  12111. + /* Release the lock only when read_depth is down to 0 */
  12112. + if (--rwlock->read_depth == 0) {
  12113. + rwlock_release(&rwlock->dep_map, 1, _RET_IP_);
  12114. + __rt_spin_unlock(&rwlock->lock);
  12115. + migrate_enable();
  12116. + }
  12117. +}
  12118. +EXPORT_SYMBOL(rt_read_unlock);
  12119. +
  12120. +unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock)
  12121. +{
  12122. + rt_write_lock(rwlock);
  12123. +
  12124. + return 0;
  12125. +}
  12126. +EXPORT_SYMBOL(rt_write_lock_irqsave);
  12127. +
  12128. +unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock)
  12129. +{
  12130. + rt_read_lock(rwlock);
  12131. +
  12132. + return 0;
  12133. +}
  12134. +EXPORT_SYMBOL(rt_read_lock_irqsave);
  12135. +
  12136. +void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key)
  12137. +{
  12138. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12139. + /*
  12140. + * Make sure we are not reinitializing a held lock:
  12141. + */
  12142. + debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
  12143. + lockdep_init_map(&rwlock->dep_map, name, key, 0);
  12144. +#endif
  12145. + rwlock->lock.save_state = 1;
  12146. + rwlock->read_depth = 0;
  12147. +}
  12148. +EXPORT_SYMBOL(__rt_rwlock_init);
  12149. +
  12150. +/*
  12151. + * rw_semaphores
  12152. + */
  12153. +
  12154. +void rt_up_write(struct rw_semaphore *rwsem)
  12155. +{
  12156. + rwsem_release(&rwsem->dep_map, 1, _RET_IP_);
  12157. + rt_mutex_unlock(&rwsem->lock);
  12158. +}
  12159. +EXPORT_SYMBOL(rt_up_write);
  12160. +
  12161. +void __rt_up_read(struct rw_semaphore *rwsem)
  12162. +{
  12163. + if (--rwsem->read_depth == 0)
  12164. + rt_mutex_unlock(&rwsem->lock);
  12165. +}
  12166. +
  12167. +void rt_up_read(struct rw_semaphore *rwsem)
  12168. +{
  12169. + rwsem_release(&rwsem->dep_map, 1, _RET_IP_);
  12170. + __rt_up_read(rwsem);
  12171. +}
  12172. +EXPORT_SYMBOL(rt_up_read);
  12173. +
  12174. +/*
  12175. + * downgrade a write lock into a read lock
  12176. + * - just wake up any readers at the front of the queue
  12177. + */
  12178. +void rt_downgrade_write(struct rw_semaphore *rwsem)
  12179. +{
  12180. + BUG_ON(rt_mutex_owner(&rwsem->lock) != current);
  12181. + rwsem->read_depth = 1;
  12182. +}
  12183. +EXPORT_SYMBOL(rt_downgrade_write);
  12184. +
  12185. +int rt_down_write_trylock(struct rw_semaphore *rwsem)
  12186. +{
  12187. + int ret = rt_mutex_trylock(&rwsem->lock);
  12188. +
  12189. + if (ret)
  12190. + rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_);
  12191. + return ret;
  12192. +}
  12193. +EXPORT_SYMBOL(rt_down_write_trylock);
  12194. +
  12195. +void rt_down_write(struct rw_semaphore *rwsem)
  12196. +{
  12197. + rwsem_acquire(&rwsem->dep_map, 0, 0, _RET_IP_);
  12198. + rt_mutex_lock(&rwsem->lock);
  12199. +}
  12200. +EXPORT_SYMBOL(rt_down_write);
  12201. +
  12202. +void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass)
  12203. +{
  12204. + rwsem_acquire(&rwsem->dep_map, subclass, 0, _RET_IP_);
  12205. + rt_mutex_lock(&rwsem->lock);
  12206. +}
  12207. +EXPORT_SYMBOL(rt_down_write_nested);
  12208. +
  12209. +void rt_down_write_nested_lock(struct rw_semaphore *rwsem,
  12210. + struct lockdep_map *nest)
  12211. +{
  12212. + rwsem_acquire_nest(&rwsem->dep_map, 0, 0, nest, _RET_IP_);
  12213. + rt_mutex_lock(&rwsem->lock);
  12214. +}
  12215. +EXPORT_SYMBOL(rt_down_write_nested_lock);
  12216. +
  12217. +int rt__down_read_trylock(struct rw_semaphore *rwsem)
  12218. +{
  12219. + struct rt_mutex *lock = &rwsem->lock;
  12220. + int ret = 1;
  12221. +
  12222. + /*
  12223. + * recursive read locks succeed when current owns the rwsem,
  12224. + * but not when read_depth == 0 which means that the rwsem is
  12225. + * write locked.
  12226. + */
  12227. + if (rt_mutex_owner(lock) != current)
  12228. + ret = rt_mutex_trylock(&rwsem->lock);
  12229. + else if (!rwsem->read_depth)
  12230. + ret = 0;
  12231. +
  12232. + if (ret)
  12233. + rwsem->read_depth++;
  12234. + return ret;
  12235. +
  12236. +}
  12237. +
  12238. +int rt_down_read_trylock(struct rw_semaphore *rwsem)
  12239. +{
  12240. + int ret;
  12241. +
  12242. + ret = rt__down_read_trylock(rwsem);
  12243. + if (ret)
  12244. + rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_);
  12245. +
  12246. + return ret;
  12247. +}
  12248. +EXPORT_SYMBOL(rt_down_read_trylock);
  12249. +
  12250. +void rt__down_read(struct rw_semaphore *rwsem)
  12251. +{
  12252. + struct rt_mutex *lock = &rwsem->lock;
  12253. +
  12254. + if (rt_mutex_owner(lock) != current)
  12255. + rt_mutex_lock(&rwsem->lock);
  12256. + rwsem->read_depth++;
  12257. +}
  12258. +EXPORT_SYMBOL(rt__down_read);
  12259. +
  12260. +static void __rt_down_read(struct rw_semaphore *rwsem, int subclass)
  12261. +{
  12262. + rwsem_acquire_read(&rwsem->dep_map, subclass, 0, _RET_IP_);
  12263. + rt__down_read(rwsem);
  12264. +}
  12265. +
  12266. +void rt_down_read(struct rw_semaphore *rwsem)
  12267. +{
  12268. + __rt_down_read(rwsem, 0);
  12269. +}
  12270. +EXPORT_SYMBOL(rt_down_read);
  12271. +
  12272. +void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass)
  12273. +{
  12274. + __rt_down_read(rwsem, subclass);
  12275. +}
  12276. +EXPORT_SYMBOL(rt_down_read_nested);
  12277. +
  12278. +void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name,
  12279. + struct lock_class_key *key)
  12280. +{
  12281. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12282. + /*
  12283. + * Make sure we are not reinitializing a held lock:
  12284. + */
  12285. + debug_check_no_locks_freed((void *)rwsem, sizeof(*rwsem));
  12286. + lockdep_init_map(&rwsem->dep_map, name, key, 0);
  12287. +#endif
  12288. + rwsem->read_depth = 0;
  12289. + rwsem->lock.save_state = 0;
  12290. +}
  12291. +EXPORT_SYMBOL(__rt_rwsem_init);
  12292. +
  12293. +/**
  12294. + * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
  12295. + * @cnt: the atomic which we are to dec
  12296. + * @lock: the mutex to return holding if we dec to 0
  12297. + *
  12298. + * return true and hold lock if we dec to 0, return false otherwise
  12299. + */
  12300. +int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
  12301. +{
  12302. + /* dec if we can't possibly hit 0 */
  12303. + if (atomic_add_unless(cnt, -1, 1))
  12304. + return 0;
  12305. + /* we might hit 0, so take the lock */
  12306. + mutex_lock(lock);
  12307. + if (!atomic_dec_and_test(cnt)) {
  12308. + /* when we actually did the dec, we didn't hit 0 */
  12309. + mutex_unlock(lock);
  12310. + return 0;
  12311. + }
  12312. + /* we hit 0, and we hold the lock */
  12313. + return 1;
  12314. +}
  12315. +EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
  12316. diff -Nur linux-4.6.6.orig/kernel/locking/rtmutex.c linux-4.6.6/kernel/locking/rtmutex.c
  12317. --- linux-4.6.6.orig/kernel/locking/rtmutex.c 2016-08-10 12:54:57.000000000 +0200
  12318. +++ linux-4.6.6/kernel/locking/rtmutex.c 2016-08-22 21:27:47.385891317 +0200
  12319. @@ -7,6 +7,11 @@
  12320. * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  12321. * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
  12322. * Copyright (C) 2006 Esben Nielsen
  12323. + * Adaptive Spinlocks:
  12324. + * Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich,
  12325. + * and Peter Morreale,
  12326. + * Adaptive Spinlocks simplification:
  12327. + * Copyright (C) 2008 Red Hat, Inc., Steven Rostedt <srostedt@redhat.com>
  12328. *
  12329. * See Documentation/locking/rt-mutex-design.txt for details.
  12330. */
  12331. @@ -16,6 +21,7 @@
  12332. #include <linux/sched/rt.h>
  12333. #include <linux/sched/deadline.h>
  12334. #include <linux/timer.h>
  12335. +#include <linux/ww_mutex.h>
  12336. #include "rtmutex_common.h"
  12337. @@ -69,6 +75,12 @@
  12338. clear_rt_mutex_waiters(lock);
  12339. }
  12340. +static int rt_mutex_real_waiter(struct rt_mutex_waiter *waiter)
  12341. +{
  12342. + return waiter && waiter != PI_WAKEUP_INPROGRESS &&
  12343. + waiter != PI_REQUEUE_INPROGRESS;
  12344. +}
  12345. +
  12346. /*
  12347. * We can speed up the acquire/release, if there's no debugging state to be
  12348. * set up.
  12349. @@ -350,6 +362,14 @@
  12350. return debug_rt_mutex_detect_deadlock(waiter, chwalk);
  12351. }
  12352. +static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter)
  12353. +{
  12354. + if (waiter->savestate)
  12355. + wake_up_lock_sleeper(waiter->task);
  12356. + else
  12357. + wake_up_process(waiter->task);
  12358. +}
  12359. +
  12360. /*
  12361. * Max number of times we'll walk the boosting chain:
  12362. */
  12363. @@ -357,7 +377,8 @@
  12364. static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
  12365. {
  12366. - return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
  12367. + return rt_mutex_real_waiter(p->pi_blocked_on) ?
  12368. + p->pi_blocked_on->lock : NULL;
  12369. }
  12370. /*
  12371. @@ -493,7 +514,7 @@
  12372. * reached or the state of the chain has changed while we
  12373. * dropped the locks.
  12374. */
  12375. - if (!waiter)
  12376. + if (!rt_mutex_real_waiter(waiter))
  12377. goto out_unlock_pi;
  12378. /*
  12379. @@ -655,13 +676,16 @@
  12380. * follow here. This is the end of the chain we are walking.
  12381. */
  12382. if (!rt_mutex_owner(lock)) {
  12383. + struct rt_mutex_waiter *lock_top_waiter;
  12384. +
  12385. /*
  12386. * If the requeue [7] above changed the top waiter,
  12387. * then we need to wake the new top waiter up to try
  12388. * to get the lock.
  12389. */
  12390. - if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
  12391. - wake_up_process(rt_mutex_top_waiter(lock)->task);
  12392. + lock_top_waiter = rt_mutex_top_waiter(lock);
  12393. + if (prerequeue_top_waiter != lock_top_waiter)
  12394. + rt_mutex_wake_waiter(lock_top_waiter);
  12395. raw_spin_unlock_irq(&lock->wait_lock);
  12396. return 0;
  12397. }
  12398. @@ -754,6 +778,25 @@
  12399. return ret;
  12400. }
  12401. +
  12402. +#define STEAL_NORMAL 0
  12403. +#define STEAL_LATERAL 1
  12404. +
  12405. +/*
  12406. + * Note that RT tasks are excluded from lateral-steals to prevent the
  12407. + * introduction of an unbounded latency
  12408. + */
  12409. +static inline int lock_is_stealable(struct task_struct *task,
  12410. + struct task_struct *pendowner, int mode)
  12411. +{
  12412. + if (mode == STEAL_NORMAL || rt_task(task)) {
  12413. + if (task->prio >= pendowner->prio)
  12414. + return 0;
  12415. + } else if (task->prio > pendowner->prio)
  12416. + return 0;
  12417. + return 1;
  12418. +}
  12419. +
  12420. /*
  12421. * Try to take an rt-mutex
  12422. *
  12423. @@ -764,8 +807,9 @@
  12424. * @waiter: The waiter that is queued to the lock's wait tree if the
  12425. * callsite called task_blocked_on_lock(), otherwise NULL
  12426. */
  12427. -static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
  12428. - struct rt_mutex_waiter *waiter)
  12429. +static int __try_to_take_rt_mutex(struct rt_mutex *lock,
  12430. + struct task_struct *task,
  12431. + struct rt_mutex_waiter *waiter, int mode)
  12432. {
  12433. /*
  12434. * Before testing whether we can acquire @lock, we set the
  12435. @@ -802,8 +846,10 @@
  12436. * If waiter is not the highest priority waiter of
  12437. * @lock, give up.
  12438. */
  12439. - if (waiter != rt_mutex_top_waiter(lock))
  12440. + if (waiter != rt_mutex_top_waiter(lock)) {
  12441. + /* XXX lock_is_stealable() ? */
  12442. return 0;
  12443. + }
  12444. /*
  12445. * We can acquire the lock. Remove the waiter from the
  12446. @@ -821,14 +867,10 @@
  12447. * not need to be dequeued.
  12448. */
  12449. if (rt_mutex_has_waiters(lock)) {
  12450. - /*
  12451. - * If @task->prio is greater than or equal to
  12452. - * the top waiter priority (kernel view),
  12453. - * @task lost.
  12454. - */
  12455. - if (task->prio >= rt_mutex_top_waiter(lock)->prio)
  12456. - return 0;
  12457. + struct task_struct *pown = rt_mutex_top_waiter(lock)->task;
  12458. + if (task != pown && !lock_is_stealable(task, pown, mode))
  12459. + return 0;
  12460. /*
  12461. * The current top waiter stays enqueued. We
  12462. * don't have to change anything in the lock
  12463. @@ -877,6 +919,399 @@
  12464. return 1;
  12465. }
  12466. +#ifdef CONFIG_PREEMPT_RT_FULL
  12467. +/*
  12468. + * preemptible spin_lock functions:
  12469. + */
  12470. +static inline void rt_spin_lock_fastlock(struct rt_mutex *lock,
  12471. + void (*slowfn)(struct rt_mutex *lock,
  12472. + bool mg_off),
  12473. + bool do_mig_dis)
  12474. +{
  12475. + might_sleep_no_state_check();
  12476. +
  12477. + if (do_mig_dis)
  12478. + migrate_disable();
  12479. +
  12480. + if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  12481. + rt_mutex_deadlock_account_lock(lock, current);
  12482. + else
  12483. + slowfn(lock, do_mig_dis);
  12484. +}
  12485. +
  12486. +static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock,
  12487. + void (*slowfn)(struct rt_mutex *lock))
  12488. +{
  12489. + if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
  12490. + rt_mutex_deadlock_account_unlock(current);
  12491. + else
  12492. + slowfn(lock);
  12493. +}
  12494. +#ifdef CONFIG_SMP
  12495. +/*
  12496. + * Note that owner is a speculative pointer and dereferencing relies
  12497. + * on rcu_read_lock() and the check against the lock owner.
  12498. + */
  12499. +static int adaptive_wait(struct rt_mutex *lock,
  12500. + struct task_struct *owner)
  12501. +{
  12502. + int res = 0;
  12503. +
  12504. + rcu_read_lock();
  12505. + for (;;) {
  12506. + if (owner != rt_mutex_owner(lock))
  12507. + break;
  12508. + /*
  12509. + * Ensure that owner->on_cpu is dereferenced _after_
  12510. + * checking the above to be valid.
  12511. + */
  12512. + barrier();
  12513. + if (!owner->on_cpu) {
  12514. + res = 1;
  12515. + break;
  12516. + }
  12517. + cpu_relax();
  12518. + }
  12519. + rcu_read_unlock();
  12520. + return res;
  12521. +}
  12522. +#else
  12523. +static int adaptive_wait(struct rt_mutex *lock,
  12524. + struct task_struct *orig_owner)
  12525. +{
  12526. + return 1;
  12527. +}
  12528. +#endif
  12529. +
  12530. +static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
  12531. + struct rt_mutex_waiter *waiter,
  12532. + struct task_struct *task,
  12533. + enum rtmutex_chainwalk chwalk);
  12534. +/*
  12535. + * Slow path lock function spin_lock style: this variant is very
  12536. + * careful not to miss any non-lock wakeups.
  12537. + *
  12538. + * We store the current state under p->pi_lock in p->saved_state and
  12539. + * the try_to_wake_up() code handles this accordingly.
  12540. + */
  12541. +static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock,
  12542. + bool mg_off)
  12543. +{
  12544. + struct task_struct *lock_owner, *self = current;
  12545. + struct rt_mutex_waiter waiter, *top_waiter;
  12546. + unsigned long flags;
  12547. + int ret;
  12548. +
  12549. + rt_mutex_init_waiter(&waiter, true);
  12550. +
  12551. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  12552. +
  12553. + if (__try_to_take_rt_mutex(lock, self, NULL, STEAL_LATERAL)) {
  12554. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12555. + return;
  12556. + }
  12557. +
  12558. + BUG_ON(rt_mutex_owner(lock) == self);
  12559. +
  12560. + /*
  12561. + * We save whatever state the task is in and we'll restore it
  12562. + * after acquiring the lock taking real wakeups into account
  12563. + * as well. We are serialized via pi_lock against wakeups. See
  12564. + * try_to_wake_up().
  12565. + */
  12566. + raw_spin_lock(&self->pi_lock);
  12567. + self->saved_state = self->state;
  12568. + __set_current_state_no_track(TASK_UNINTERRUPTIBLE);
  12569. + raw_spin_unlock(&self->pi_lock);
  12570. +
  12571. + ret = task_blocks_on_rt_mutex(lock, &waiter, self, RT_MUTEX_MIN_CHAINWALK);
  12572. + BUG_ON(ret);
  12573. +
  12574. + for (;;) {
  12575. + /* Try to acquire the lock again. */
  12576. + if (__try_to_take_rt_mutex(lock, self, &waiter, STEAL_LATERAL))
  12577. + break;
  12578. +
  12579. + top_waiter = rt_mutex_top_waiter(lock);
  12580. + lock_owner = rt_mutex_owner(lock);
  12581. +
  12582. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12583. +
  12584. + debug_rt_mutex_print_deadlock(&waiter);
  12585. +
  12586. + if (top_waiter != &waiter || adaptive_wait(lock, lock_owner)) {
  12587. + if (mg_off)
  12588. + migrate_enable();
  12589. + schedule();
  12590. + if (mg_off)
  12591. + migrate_disable();
  12592. + }
  12593. +
  12594. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  12595. +
  12596. + raw_spin_lock(&self->pi_lock);
  12597. + __set_current_state_no_track(TASK_UNINTERRUPTIBLE);
  12598. + raw_spin_unlock(&self->pi_lock);
  12599. + }
  12600. +
  12601. + /*
  12602. + * Restore the task state to current->saved_state. We set it
  12603. + * to the original state above and the try_to_wake_up() code
  12604. + * has possibly updated it when a real (non-rtmutex) wakeup
  12605. + * happened while we were blocked. Clear saved_state so
  12606. + * try_to_wakeup() does not get confused.
  12607. + */
  12608. + raw_spin_lock(&self->pi_lock);
  12609. + __set_current_state_no_track(self->saved_state);
  12610. + self->saved_state = TASK_RUNNING;
  12611. + raw_spin_unlock(&self->pi_lock);
  12612. +
  12613. + /*
  12614. + * try_to_take_rt_mutex() sets the waiter bit
  12615. + * unconditionally. We might have to fix that up:
  12616. + */
  12617. + fixup_rt_mutex_waiters(lock);
  12618. +
  12619. + BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock));
  12620. + BUG_ON(!RB_EMPTY_NODE(&waiter.tree_entry));
  12621. +
  12622. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12623. +
  12624. + debug_rt_mutex_free_waiter(&waiter);
  12625. +}
  12626. +
  12627. +static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
  12628. + struct wake_q_head *wake_sleeper_q,
  12629. + struct rt_mutex *lock);
  12630. +/*
  12631. + * Slow path to release a rt_mutex spin_lock style
  12632. + */
  12633. +static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock)
  12634. +{
  12635. + unsigned long flags;
  12636. + WAKE_Q(wake_q);
  12637. + WAKE_Q(wake_sleeper_q);
  12638. +
  12639. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  12640. +
  12641. + debug_rt_mutex_unlock(lock);
  12642. +
  12643. + rt_mutex_deadlock_account_unlock(current);
  12644. +
  12645. + if (!rt_mutex_has_waiters(lock)) {
  12646. + lock->owner = NULL;
  12647. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12648. + return;
  12649. + }
  12650. +
  12651. + mark_wakeup_next_waiter(&wake_q, &wake_sleeper_q, lock);
  12652. +
  12653. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12654. + wake_up_q(&wake_q);
  12655. + wake_up_q_sleeper(&wake_sleeper_q);
  12656. +
  12657. + /* Undo pi boosting.when necessary */
  12658. + rt_mutex_adjust_prio(current);
  12659. +}
  12660. +
  12661. +void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock)
  12662. +{
  12663. + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, false);
  12664. + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12665. +}
  12666. +EXPORT_SYMBOL(rt_spin_lock__no_mg);
  12667. +
  12668. +void __lockfunc rt_spin_lock(spinlock_t *lock)
  12669. +{
  12670. + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, true);
  12671. + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12672. +}
  12673. +EXPORT_SYMBOL(rt_spin_lock);
  12674. +
  12675. +void __lockfunc __rt_spin_lock(struct rt_mutex *lock)
  12676. +{
  12677. + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock, true);
  12678. +}
  12679. +EXPORT_SYMBOL(__rt_spin_lock);
  12680. +
  12681. +void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock)
  12682. +{
  12683. + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock, false);
  12684. +}
  12685. +EXPORT_SYMBOL(__rt_spin_lock__no_mg);
  12686. +
  12687. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12688. +void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass)
  12689. +{
  12690. + spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  12691. + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, true);
  12692. +}
  12693. +EXPORT_SYMBOL(rt_spin_lock_nested);
  12694. +#endif
  12695. +
  12696. +void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock)
  12697. +{
  12698. + /* NOTE: we always pass in '1' for nested, for simplicity */
  12699. + spin_release(&lock->dep_map, 1, _RET_IP_);
  12700. + rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock);
  12701. +}
  12702. +EXPORT_SYMBOL(rt_spin_unlock__no_mg);
  12703. +
  12704. +void __lockfunc rt_spin_unlock(spinlock_t *lock)
  12705. +{
  12706. + /* NOTE: we always pass in '1' for nested, for simplicity */
  12707. + spin_release(&lock->dep_map, 1, _RET_IP_);
  12708. + rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock);
  12709. + migrate_enable();
  12710. +}
  12711. +EXPORT_SYMBOL(rt_spin_unlock);
  12712. +
  12713. +void __lockfunc __rt_spin_unlock(struct rt_mutex *lock)
  12714. +{
  12715. + rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock);
  12716. +}
  12717. +EXPORT_SYMBOL(__rt_spin_unlock);
  12718. +
  12719. +/*
  12720. + * Wait for the lock to get unlocked: instead of polling for an unlock
  12721. + * (like raw spinlocks do), we lock and unlock, to force the kernel to
  12722. + * schedule if there's contention:
  12723. + */
  12724. +void __lockfunc rt_spin_unlock_wait(spinlock_t *lock)
  12725. +{
  12726. + spin_lock(lock);
  12727. + spin_unlock(lock);
  12728. +}
  12729. +EXPORT_SYMBOL(rt_spin_unlock_wait);
  12730. +
  12731. +int __lockfunc __rt_spin_trylock(struct rt_mutex *lock)
  12732. +{
  12733. + return rt_mutex_trylock(lock);
  12734. +}
  12735. +
  12736. +int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock)
  12737. +{
  12738. + int ret;
  12739. +
  12740. + ret = rt_mutex_trylock(&lock->lock);
  12741. + if (ret)
  12742. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12743. + return ret;
  12744. +}
  12745. +EXPORT_SYMBOL(rt_spin_trylock__no_mg);
  12746. +
  12747. +int __lockfunc rt_spin_trylock(spinlock_t *lock)
  12748. +{
  12749. + int ret;
  12750. +
  12751. + migrate_disable();
  12752. + ret = rt_mutex_trylock(&lock->lock);
  12753. + if (ret)
  12754. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12755. + else
  12756. + migrate_enable();
  12757. + return ret;
  12758. +}
  12759. +EXPORT_SYMBOL(rt_spin_trylock);
  12760. +
  12761. +int __lockfunc rt_spin_trylock_bh(spinlock_t *lock)
  12762. +{
  12763. + int ret;
  12764. +
  12765. + local_bh_disable();
  12766. + ret = rt_mutex_trylock(&lock->lock);
  12767. + if (ret) {
  12768. + migrate_disable();
  12769. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12770. + } else
  12771. + local_bh_enable();
  12772. + return ret;
  12773. +}
  12774. +EXPORT_SYMBOL(rt_spin_trylock_bh);
  12775. +
  12776. +int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags)
  12777. +{
  12778. + int ret;
  12779. +
  12780. + *flags = 0;
  12781. + ret = rt_mutex_trylock(&lock->lock);
  12782. + if (ret) {
  12783. + migrate_disable();
  12784. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12785. + }
  12786. + return ret;
  12787. +}
  12788. +EXPORT_SYMBOL(rt_spin_trylock_irqsave);
  12789. +
  12790. +int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock)
  12791. +{
  12792. + /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
  12793. + if (atomic_add_unless(atomic, -1, 1))
  12794. + return 0;
  12795. + rt_spin_lock(lock);
  12796. + if (atomic_dec_and_test(atomic))
  12797. + return 1;
  12798. + rt_spin_unlock(lock);
  12799. + return 0;
  12800. +}
  12801. +EXPORT_SYMBOL(atomic_dec_and_spin_lock);
  12802. +
  12803. + void
  12804. +__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key)
  12805. +{
  12806. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12807. + /*
  12808. + * Make sure we are not reinitializing a held lock:
  12809. + */
  12810. + debug_check_no_locks_freed((void *)lock, sizeof(*lock));
  12811. + lockdep_init_map(&lock->dep_map, name, key, 0);
  12812. +#endif
  12813. +}
  12814. +EXPORT_SYMBOL(__rt_spin_lock_init);
  12815. +
  12816. +#endif /* PREEMPT_RT_FULL */
  12817. +
  12818. +#ifdef CONFIG_PREEMPT_RT_FULL
  12819. + static inline int __sched
  12820. +__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx)
  12821. +{
  12822. + struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock);
  12823. + struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
  12824. +
  12825. + if (!hold_ctx)
  12826. + return 0;
  12827. +
  12828. + if (unlikely(ctx == hold_ctx))
  12829. + return -EALREADY;
  12830. +
  12831. + if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
  12832. + (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
  12833. +#ifdef CONFIG_DEBUG_MUTEXES
  12834. + DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
  12835. + ctx->contending_lock = ww;
  12836. +#endif
  12837. + return -EDEADLK;
  12838. + }
  12839. +
  12840. + return 0;
  12841. +}
  12842. +#else
  12843. + static inline int __sched
  12844. +__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx)
  12845. +{
  12846. + BUG();
  12847. + return 0;
  12848. +}
  12849. +
  12850. +#endif
  12851. +
  12852. +static inline int
  12853. +try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
  12854. + struct rt_mutex_waiter *waiter)
  12855. +{
  12856. + return __try_to_take_rt_mutex(lock, task, waiter, STEAL_NORMAL);
  12857. +}
  12858. +
  12859. /*
  12860. * Task blocks on lock.
  12861. *
  12862. @@ -907,6 +1342,23 @@
  12863. return -EDEADLK;
  12864. raw_spin_lock(&task->pi_lock);
  12865. +
  12866. + /*
  12867. + * In the case of futex requeue PI, this will be a proxy
  12868. + * lock. The task will wake unaware that it is enqueueed on
  12869. + * this lock. Avoid blocking on two locks and corrupting
  12870. + * pi_blocked_on via the PI_WAKEUP_INPROGRESS
  12871. + * flag. futex_wait_requeue_pi() sets this when it wakes up
  12872. + * before requeue (due to a signal or timeout). Do not enqueue
  12873. + * the task if PI_WAKEUP_INPROGRESS is set.
  12874. + */
  12875. + if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) {
  12876. + raw_spin_unlock(&task->pi_lock);
  12877. + return -EAGAIN;
  12878. + }
  12879. +
  12880. + BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on));
  12881. +
  12882. __rt_mutex_adjust_prio(task);
  12883. waiter->task = task;
  12884. waiter->lock = lock;
  12885. @@ -930,7 +1382,7 @@
  12886. rt_mutex_enqueue_pi(owner, waiter);
  12887. __rt_mutex_adjust_prio(owner);
  12888. - if (owner->pi_blocked_on)
  12889. + if (rt_mutex_real_waiter(owner->pi_blocked_on))
  12890. chain_walk = 1;
  12891. } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
  12892. chain_walk = 1;
  12893. @@ -972,6 +1424,7 @@
  12894. * Called with lock->wait_lock held and interrupts disabled.
  12895. */
  12896. static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
  12897. + struct wake_q_head *wake_sleeper_q,
  12898. struct rt_mutex *lock)
  12899. {
  12900. struct rt_mutex_waiter *waiter;
  12901. @@ -1000,7 +1453,10 @@
  12902. raw_spin_unlock(&current->pi_lock);
  12903. - wake_q_add(wake_q, waiter->task);
  12904. + if (waiter->savestate)
  12905. + wake_q_add(wake_sleeper_q, waiter->task);
  12906. + else
  12907. + wake_q_add(wake_q, waiter->task);
  12908. }
  12909. /*
  12910. @@ -1014,7 +1470,7 @@
  12911. {
  12912. bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
  12913. struct task_struct *owner = rt_mutex_owner(lock);
  12914. - struct rt_mutex *next_lock;
  12915. + struct rt_mutex *next_lock = NULL;
  12916. raw_spin_lock(&current->pi_lock);
  12917. rt_mutex_dequeue(lock, waiter);
  12918. @@ -1038,7 +1494,8 @@
  12919. __rt_mutex_adjust_prio(owner);
  12920. /* Store the lock on which owner is blocked or NULL */
  12921. - next_lock = task_blocked_on_lock(owner);
  12922. + if (rt_mutex_real_waiter(owner->pi_blocked_on))
  12923. + next_lock = task_blocked_on_lock(owner);
  12924. raw_spin_unlock(&owner->pi_lock);
  12925. @@ -1074,17 +1531,17 @@
  12926. raw_spin_lock_irqsave(&task->pi_lock, flags);
  12927. waiter = task->pi_blocked_on;
  12928. - if (!waiter || (waiter->prio == task->prio &&
  12929. + if (!rt_mutex_real_waiter(waiter) || (waiter->prio == task->prio &&
  12930. !dl_prio(task->prio))) {
  12931. raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  12932. return;
  12933. }
  12934. next_lock = waiter->lock;
  12935. - raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  12936. /* gets dropped in rt_mutex_adjust_prio_chain()! */
  12937. get_task_struct(task);
  12938. + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  12939. rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
  12940. next_lock, NULL, task);
  12941. }
  12942. @@ -1102,7 +1559,8 @@
  12943. static int __sched
  12944. __rt_mutex_slowlock(struct rt_mutex *lock, int state,
  12945. struct hrtimer_sleeper *timeout,
  12946. - struct rt_mutex_waiter *waiter)
  12947. + struct rt_mutex_waiter *waiter,
  12948. + struct ww_acquire_ctx *ww_ctx)
  12949. {
  12950. int ret = 0;
  12951. @@ -1125,6 +1583,12 @@
  12952. break;
  12953. }
  12954. + if (ww_ctx && ww_ctx->acquired > 0) {
  12955. + ret = __mutex_lock_check_stamp(lock, ww_ctx);
  12956. + if (ret)
  12957. + break;
  12958. + }
  12959. +
  12960. raw_spin_unlock_irq(&lock->wait_lock);
  12961. debug_rt_mutex_print_deadlock(waiter);
  12962. @@ -1159,21 +1623,96 @@
  12963. }
  12964. }
  12965. +static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
  12966. + struct ww_acquire_ctx *ww_ctx)
  12967. +{
  12968. +#ifdef CONFIG_DEBUG_MUTEXES
  12969. + /*
  12970. + * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
  12971. + * but released with a normal mutex_unlock in this call.
  12972. + *
  12973. + * This should never happen, always use ww_mutex_unlock.
  12974. + */
  12975. + DEBUG_LOCKS_WARN_ON(ww->ctx);
  12976. +
  12977. + /*
  12978. + * Not quite done after calling ww_acquire_done() ?
  12979. + */
  12980. + DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
  12981. +
  12982. + if (ww_ctx->contending_lock) {
  12983. + /*
  12984. + * After -EDEADLK you tried to
  12985. + * acquire a different ww_mutex? Bad!
  12986. + */
  12987. + DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
  12988. +
  12989. + /*
  12990. + * You called ww_mutex_lock after receiving -EDEADLK,
  12991. + * but 'forgot' to unlock everything else first?
  12992. + */
  12993. + DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
  12994. + ww_ctx->contending_lock = NULL;
  12995. + }
  12996. +
  12997. + /*
  12998. + * Naughty, using a different class will lead to undefined behavior!
  12999. + */
  13000. + DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
  13001. +#endif
  13002. + ww_ctx->acquired++;
  13003. +}
  13004. +
  13005. +#ifdef CONFIG_PREEMPT_RT_FULL
  13006. +static void ww_mutex_account_lock(struct rt_mutex *lock,
  13007. + struct ww_acquire_ctx *ww_ctx)
  13008. +{
  13009. + struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock);
  13010. + struct rt_mutex_waiter *waiter, *n;
  13011. +
  13012. + /*
  13013. + * This branch gets optimized out for the common case,
  13014. + * and is only important for ww_mutex_lock.
  13015. + */
  13016. + ww_mutex_lock_acquired(ww, ww_ctx);
  13017. + ww->ctx = ww_ctx;
  13018. +
  13019. + /*
  13020. + * Give any possible sleeping processes the chance to wake up,
  13021. + * so they can recheck if they have to back off.
  13022. + */
  13023. + rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters,
  13024. + tree_entry) {
  13025. + /* XXX debug rt mutex waiter wakeup */
  13026. +
  13027. + BUG_ON(waiter->lock != lock);
  13028. + rt_mutex_wake_waiter(waiter);
  13029. + }
  13030. +}
  13031. +
  13032. +#else
  13033. +
  13034. +static void ww_mutex_account_lock(struct rt_mutex *lock,
  13035. + struct ww_acquire_ctx *ww_ctx)
  13036. +{
  13037. + BUG();
  13038. +}
  13039. +#endif
  13040. +
  13041. /*
  13042. * Slow path lock function:
  13043. */
  13044. static int __sched
  13045. rt_mutex_slowlock(struct rt_mutex *lock, int state,
  13046. struct hrtimer_sleeper *timeout,
  13047. - enum rtmutex_chainwalk chwalk)
  13048. + enum rtmutex_chainwalk chwalk,
  13049. + struct ww_acquire_ctx *ww_ctx)
  13050. {
  13051. struct rt_mutex_waiter waiter;
  13052. unsigned long flags;
  13053. int ret = 0;
  13054. - debug_rt_mutex_init_waiter(&waiter);
  13055. - RB_CLEAR_NODE(&waiter.pi_tree_entry);
  13056. - RB_CLEAR_NODE(&waiter.tree_entry);
  13057. + rt_mutex_init_waiter(&waiter, false);
  13058. /*
  13059. * Technically we could use raw_spin_[un]lock_irq() here, but this can
  13060. @@ -1187,6 +1726,8 @@
  13061. /* Try to acquire the lock again: */
  13062. if (try_to_take_rt_mutex(lock, current, NULL)) {
  13063. + if (ww_ctx)
  13064. + ww_mutex_account_lock(lock, ww_ctx);
  13065. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  13066. return 0;
  13067. }
  13068. @@ -1201,13 +1742,23 @@
  13069. if (likely(!ret))
  13070. /* sleep on the mutex */
  13071. - ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
  13072. + ret = __rt_mutex_slowlock(lock, state, timeout, &waiter,
  13073. + ww_ctx);
  13074. + else if (ww_ctx) {
  13075. + /* ww_mutex received EDEADLK, let it become EALREADY */
  13076. + ret = __mutex_lock_check_stamp(lock, ww_ctx);
  13077. + BUG_ON(!ret);
  13078. + }
  13079. if (unlikely(ret)) {
  13080. __set_current_state(TASK_RUNNING);
  13081. if (rt_mutex_has_waiters(lock))
  13082. remove_waiter(lock, &waiter);
  13083. - rt_mutex_handle_deadlock(ret, chwalk, &waiter);
  13084. + /* ww_mutex want to report EDEADLK/EALREADY, let them */
  13085. + if (!ww_ctx)
  13086. + rt_mutex_handle_deadlock(ret, chwalk, &waiter);
  13087. + } else if (ww_ctx) {
  13088. + ww_mutex_account_lock(lock, ww_ctx);
  13089. }
  13090. /*
  13091. @@ -1267,7 +1818,8 @@
  13092. * Return whether the current task needs to undo a potential priority boosting.
  13093. */
  13094. static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
  13095. - struct wake_q_head *wake_q)
  13096. + struct wake_q_head *wake_q,
  13097. + struct wake_q_head *wake_sleeper_q)
  13098. {
  13099. unsigned long flags;
  13100. @@ -1323,7 +1875,7 @@
  13101. *
  13102. * Queue the next waiter for wakeup once we release the wait_lock.
  13103. */
  13104. - mark_wakeup_next_waiter(wake_q, lock);
  13105. + mark_wakeup_next_waiter(wake_q, wake_sleeper_q, lock);
  13106. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  13107. @@ -1339,31 +1891,36 @@
  13108. */
  13109. static inline int
  13110. rt_mutex_fastlock(struct rt_mutex *lock, int state,
  13111. + struct ww_acquire_ctx *ww_ctx,
  13112. int (*slowfn)(struct rt_mutex *lock, int state,
  13113. struct hrtimer_sleeper *timeout,
  13114. - enum rtmutex_chainwalk chwalk))
  13115. + enum rtmutex_chainwalk chwalk,
  13116. + struct ww_acquire_ctx *ww_ctx))
  13117. {
  13118. if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {
  13119. rt_mutex_deadlock_account_lock(lock, current);
  13120. return 0;
  13121. } else
  13122. - return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
  13123. + return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK,
  13124. + ww_ctx);
  13125. }
  13126. static inline int
  13127. rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
  13128. struct hrtimer_sleeper *timeout,
  13129. enum rtmutex_chainwalk chwalk,
  13130. + struct ww_acquire_ctx *ww_ctx,
  13131. int (*slowfn)(struct rt_mutex *lock, int state,
  13132. struct hrtimer_sleeper *timeout,
  13133. - enum rtmutex_chainwalk chwalk))
  13134. + enum rtmutex_chainwalk chwalk,
  13135. + struct ww_acquire_ctx *ww_ctx))
  13136. {
  13137. if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
  13138. likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {
  13139. rt_mutex_deadlock_account_lock(lock, current);
  13140. return 0;
  13141. } else
  13142. - return slowfn(lock, state, timeout, chwalk);
  13143. + return slowfn(lock, state, timeout, chwalk, ww_ctx);
  13144. }
  13145. static inline int
  13146. @@ -1380,17 +1937,20 @@
  13147. static inline void
  13148. rt_mutex_fastunlock(struct rt_mutex *lock,
  13149. bool (*slowfn)(struct rt_mutex *lock,
  13150. - struct wake_q_head *wqh))
  13151. + struct wake_q_head *wqh,
  13152. + struct wake_q_head *wq_sleeper))
  13153. {
  13154. WAKE_Q(wake_q);
  13155. + WAKE_Q(wake_sleeper_q);
  13156. if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {
  13157. rt_mutex_deadlock_account_unlock(current);
  13158. } else {
  13159. - bool deboost = slowfn(lock, &wake_q);
  13160. + bool deboost = slowfn(lock, &wake_q, &wake_sleeper_q);
  13161. wake_up_q(&wake_q);
  13162. + wake_up_q_sleeper(&wake_sleeper_q);
  13163. /* Undo pi boosting if necessary: */
  13164. if (deboost)
  13165. @@ -1407,7 +1967,7 @@
  13166. {
  13167. might_sleep();
  13168. - rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
  13169. + rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, NULL, rt_mutex_slowlock);
  13170. }
  13171. EXPORT_SYMBOL_GPL(rt_mutex_lock);
  13172. @@ -1424,7 +1984,7 @@
  13173. {
  13174. might_sleep();
  13175. - return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
  13176. + return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, NULL, rt_mutex_slowlock);
  13177. }
  13178. EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
  13179. @@ -1437,11 +1997,30 @@
  13180. might_sleep();
  13181. return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  13182. - RT_MUTEX_FULL_CHAINWALK,
  13183. + RT_MUTEX_FULL_CHAINWALK, NULL,
  13184. rt_mutex_slowlock);
  13185. }
  13186. /**
  13187. + * rt_mutex_lock_killable - lock a rt_mutex killable
  13188. + *
  13189. + * @lock: the rt_mutex to be locked
  13190. + * @detect_deadlock: deadlock detection on/off
  13191. + *
  13192. + * Returns:
  13193. + * 0 on success
  13194. + * -EINTR when interrupted by a signal
  13195. + * -EDEADLK when the lock would deadlock (when deadlock detection is on)
  13196. + */
  13197. +int __sched rt_mutex_lock_killable(struct rt_mutex *lock)
  13198. +{
  13199. + might_sleep();
  13200. +
  13201. + return rt_mutex_fastlock(lock, TASK_KILLABLE, NULL, rt_mutex_slowlock);
  13202. +}
  13203. +EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
  13204. +
  13205. +/**
  13206. * rt_mutex_timed_lock - lock a rt_mutex interruptible
  13207. * the timeout structure is provided
  13208. * by the caller
  13209. @@ -1461,6 +2040,7 @@
  13210. return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  13211. RT_MUTEX_MIN_CHAINWALK,
  13212. + NULL,
  13213. rt_mutex_slowlock);
  13214. }
  13215. EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
  13216. @@ -1478,7 +2058,11 @@
  13217. */
  13218. int __sched rt_mutex_trylock(struct rt_mutex *lock)
  13219. {
  13220. +#ifdef CONFIG_PREEMPT_RT_FULL
  13221. + if (WARN_ON_ONCE(in_irq() || in_nmi()))
  13222. +#else
  13223. if (WARN_ON(in_irq() || in_nmi() || in_serving_softirq()))
  13224. +#endif
  13225. return 0;
  13226. return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
  13227. @@ -1504,13 +2088,14 @@
  13228. * required or not.
  13229. */
  13230. bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock,
  13231. - struct wake_q_head *wqh)
  13232. + struct wake_q_head *wqh,
  13233. + struct wake_q_head *wq_sleeper)
  13234. {
  13235. if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {
  13236. rt_mutex_deadlock_account_unlock(current);
  13237. return false;
  13238. }
  13239. - return rt_mutex_slowunlock(lock, wqh);
  13240. + return rt_mutex_slowunlock(lock, wqh, wq_sleeper);
  13241. }
  13242. /**
  13243. @@ -1543,13 +2128,12 @@
  13244. void __rt_mutex_init(struct rt_mutex *lock, const char *name)
  13245. {
  13246. lock->owner = NULL;
  13247. - raw_spin_lock_init(&lock->wait_lock);
  13248. lock->waiters = RB_ROOT;
  13249. lock->waiters_leftmost = NULL;
  13250. debug_rt_mutex_init(lock, name);
  13251. }
  13252. -EXPORT_SYMBOL_GPL(__rt_mutex_init);
  13253. +EXPORT_SYMBOL(__rt_mutex_init);
  13254. /**
  13255. * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
  13256. @@ -1564,7 +2148,7 @@
  13257. void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  13258. struct task_struct *proxy_owner)
  13259. {
  13260. - __rt_mutex_init(lock, NULL);
  13261. + rt_mutex_init(lock);
  13262. debug_rt_mutex_proxy_lock(lock, proxy_owner);
  13263. rt_mutex_set_owner(lock, proxy_owner);
  13264. rt_mutex_deadlock_account_lock(lock, proxy_owner);
  13265. @@ -1612,6 +2196,35 @@
  13266. return 1;
  13267. }
  13268. +#ifdef CONFIG_PREEMPT_RT_FULL
  13269. + /*
  13270. + * In PREEMPT_RT there's an added race.
  13271. + * If the task, that we are about to requeue, times out,
  13272. + * it can set the PI_WAKEUP_INPROGRESS. This tells the requeue
  13273. + * to skip this task. But right after the task sets
  13274. + * its pi_blocked_on to PI_WAKEUP_INPROGRESS it can then
  13275. + * block on the spin_lock(&hb->lock), which in RT is an rtmutex.
  13276. + * This will replace the PI_WAKEUP_INPROGRESS with the actual
  13277. + * lock that it blocks on. We *must not* place this task
  13278. + * on this proxy lock in that case.
  13279. + *
  13280. + * To prevent this race, we first take the task's pi_lock
  13281. + * and check if it has updated its pi_blocked_on. If it has,
  13282. + * we assume that it woke up and we return -EAGAIN.
  13283. + * Otherwise, we set the task's pi_blocked_on to
  13284. + * PI_REQUEUE_INPROGRESS, so that if the task is waking up
  13285. + * it will know that we are in the process of requeuing it.
  13286. + */
  13287. + raw_spin_lock(&task->pi_lock);
  13288. + if (task->pi_blocked_on) {
  13289. + raw_spin_unlock(&task->pi_lock);
  13290. + raw_spin_unlock_irq(&lock->wait_lock);
  13291. + return -EAGAIN;
  13292. + }
  13293. + task->pi_blocked_on = PI_REQUEUE_INPROGRESS;
  13294. + raw_spin_unlock(&task->pi_lock);
  13295. +#endif
  13296. +
  13297. /* We enforce deadlock detection for futexes */
  13298. ret = task_blocks_on_rt_mutex(lock, waiter, task,
  13299. RT_MUTEX_FULL_CHAINWALK);
  13300. @@ -1626,7 +2239,7 @@
  13301. ret = 0;
  13302. }
  13303. - if (unlikely(ret))
  13304. + if (ret && rt_mutex_has_waiters(lock))
  13305. remove_waiter(lock, waiter);
  13306. raw_spin_unlock_irq(&lock->wait_lock);
  13307. @@ -1682,7 +2295,7 @@
  13308. set_current_state(TASK_INTERRUPTIBLE);
  13309. /* sleep on the mutex */
  13310. - ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
  13311. + ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL);
  13312. if (unlikely(ret))
  13313. remove_waiter(lock, waiter);
  13314. @@ -1697,3 +2310,89 @@
  13315. return ret;
  13316. }
  13317. +
  13318. +static inline int
  13319. +ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  13320. +{
  13321. +#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
  13322. + unsigned tmp;
  13323. +
  13324. + if (ctx->deadlock_inject_countdown-- == 0) {
  13325. + tmp = ctx->deadlock_inject_interval;
  13326. + if (tmp > UINT_MAX/4)
  13327. + tmp = UINT_MAX;
  13328. + else
  13329. + tmp = tmp*2 + tmp + tmp/2;
  13330. +
  13331. + ctx->deadlock_inject_interval = tmp;
  13332. + ctx->deadlock_inject_countdown = tmp;
  13333. + ctx->contending_lock = lock;
  13334. +
  13335. + ww_mutex_unlock(lock);
  13336. +
  13337. + return -EDEADLK;
  13338. + }
  13339. +#endif
  13340. +
  13341. + return 0;
  13342. +}
  13343. +
  13344. +#ifdef CONFIG_PREEMPT_RT_FULL
  13345. +int __sched
  13346. +__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx)
  13347. +{
  13348. + int ret;
  13349. +
  13350. + might_sleep();
  13351. +
  13352. + mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_);
  13353. + ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx);
  13354. + if (ret)
  13355. + mutex_release(&lock->base.dep_map, 1, _RET_IP_);
  13356. + else if (!ret && ww_ctx->acquired > 1)
  13357. + return ww_mutex_deadlock_injection(lock, ww_ctx);
  13358. +
  13359. + return ret;
  13360. +}
  13361. +EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
  13362. +
  13363. +int __sched
  13364. +__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx)
  13365. +{
  13366. + int ret;
  13367. +
  13368. + might_sleep();
  13369. +
  13370. + mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_);
  13371. + ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx);
  13372. + if (ret)
  13373. + mutex_release(&lock->base.dep_map, 1, _RET_IP_);
  13374. + else if (!ret && ww_ctx->acquired > 1)
  13375. + return ww_mutex_deadlock_injection(lock, ww_ctx);
  13376. +
  13377. + return ret;
  13378. +}
  13379. +EXPORT_SYMBOL_GPL(__ww_mutex_lock);
  13380. +
  13381. +void __sched ww_mutex_unlock(struct ww_mutex *lock)
  13382. +{
  13383. + int nest = !!lock->ctx;
  13384. +
  13385. + /*
  13386. + * The unlocking fastpath is the 0->1 transition from 'locked'
  13387. + * into 'unlocked' state:
  13388. + */
  13389. + if (nest) {
  13390. +#ifdef CONFIG_DEBUG_MUTEXES
  13391. + DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
  13392. +#endif
  13393. + if (lock->ctx->acquired > 0)
  13394. + lock->ctx->acquired--;
  13395. + lock->ctx = NULL;
  13396. + }
  13397. +
  13398. + mutex_release(&lock->base.dep_map, nest, _RET_IP_);
  13399. + rt_mutex_unlock(&lock->base.lock);
  13400. +}
  13401. +EXPORT_SYMBOL(ww_mutex_unlock);
  13402. +#endif
  13403. diff -Nur linux-4.6.6.orig/kernel/locking/rtmutex_common.h linux-4.6.6/kernel/locking/rtmutex_common.h
  13404. --- linux-4.6.6.orig/kernel/locking/rtmutex_common.h 2016-08-10 12:54:57.000000000 +0200
  13405. +++ linux-4.6.6/kernel/locking/rtmutex_common.h 2016-08-22 21:27:47.385891317 +0200
  13406. @@ -27,6 +27,7 @@
  13407. struct rb_node pi_tree_entry;
  13408. struct task_struct *task;
  13409. struct rt_mutex *lock;
  13410. + bool savestate;
  13411. #ifdef CONFIG_DEBUG_RT_MUTEXES
  13412. unsigned long ip;
  13413. struct pid *deadlock_task_pid;
  13414. @@ -97,6 +98,9 @@
  13415. /*
  13416. * PI-futex support (proxy locking functions, etc.):
  13417. */
  13418. +#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1)
  13419. +#define PI_REQUEUE_INPROGRESS ((struct rt_mutex_waiter *) 2)
  13420. +
  13421. extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock);
  13422. extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  13423. struct task_struct *proxy_owner);
  13424. @@ -110,7 +114,8 @@
  13425. struct rt_mutex_waiter *waiter);
  13426. extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
  13427. extern bool rt_mutex_futex_unlock(struct rt_mutex *lock,
  13428. - struct wake_q_head *wqh);
  13429. + struct wake_q_head *wqh,
  13430. + struct wake_q_head *wq_sleeper);
  13431. extern void rt_mutex_adjust_prio(struct task_struct *task);
  13432. #ifdef CONFIG_DEBUG_RT_MUTEXES
  13433. @@ -119,4 +124,14 @@
  13434. # include "rtmutex.h"
  13435. #endif
  13436. +static inline void
  13437. +rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savestate)
  13438. +{
  13439. + debug_rt_mutex_init_waiter(waiter);
  13440. + waiter->task = NULL;
  13441. + waiter->savestate = savestate;
  13442. + RB_CLEAR_NODE(&waiter->pi_tree_entry);
  13443. + RB_CLEAR_NODE(&waiter->tree_entry);
  13444. +}
  13445. +
  13446. #endif
  13447. diff -Nur linux-4.6.6.orig/kernel/locking/spinlock.c linux-4.6.6/kernel/locking/spinlock.c
  13448. --- linux-4.6.6.orig/kernel/locking/spinlock.c 2016-08-10 12:54:57.000000000 +0200
  13449. +++ linux-4.6.6/kernel/locking/spinlock.c 2016-08-22 21:27:47.389891577 +0200
  13450. @@ -124,8 +124,11 @@
  13451. * __[spin|read|write]_lock_bh()
  13452. */
  13453. BUILD_LOCK_OPS(spin, raw_spinlock);
  13454. +
  13455. +#ifndef CONFIG_PREEMPT_RT_FULL
  13456. BUILD_LOCK_OPS(read, rwlock);
  13457. BUILD_LOCK_OPS(write, rwlock);
  13458. +#endif
  13459. #endif
  13460. @@ -209,6 +212,8 @@
  13461. EXPORT_SYMBOL(_raw_spin_unlock_bh);
  13462. #endif
  13463. +#ifndef CONFIG_PREEMPT_RT_FULL
  13464. +
  13465. #ifndef CONFIG_INLINE_READ_TRYLOCK
  13466. int __lockfunc _raw_read_trylock(rwlock_t *lock)
  13467. {
  13468. @@ -353,6 +358,8 @@
  13469. EXPORT_SYMBOL(_raw_write_unlock_bh);
  13470. #endif
  13471. +#endif /* !PREEMPT_RT_FULL */
  13472. +
  13473. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  13474. void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
  13475. diff -Nur linux-4.6.6.orig/kernel/locking/spinlock_debug.c linux-4.6.6/kernel/locking/spinlock_debug.c
  13476. --- linux-4.6.6.orig/kernel/locking/spinlock_debug.c 2016-08-10 12:54:57.000000000 +0200
  13477. +++ linux-4.6.6/kernel/locking/spinlock_debug.c 2016-08-22 21:27:47.389891577 +0200
  13478. @@ -31,6 +31,7 @@
  13479. EXPORT_SYMBOL(__raw_spin_lock_init);
  13480. +#ifndef CONFIG_PREEMPT_RT_FULL
  13481. void __rwlock_init(rwlock_t *lock, const char *name,
  13482. struct lock_class_key *key)
  13483. {
  13484. @@ -48,6 +49,7 @@
  13485. }
  13486. EXPORT_SYMBOL(__rwlock_init);
  13487. +#endif
  13488. static void spin_dump(raw_spinlock_t *lock, const char *msg)
  13489. {
  13490. @@ -159,6 +161,7 @@
  13491. arch_spin_unlock(&lock->raw_lock);
  13492. }
  13493. +#ifndef CONFIG_PREEMPT_RT_FULL
  13494. static void rwlock_bug(rwlock_t *lock, const char *msg)
  13495. {
  13496. if (!debug_locks_off())
  13497. @@ -300,3 +303,5 @@
  13498. debug_write_unlock(lock);
  13499. arch_write_unlock(&lock->raw_lock);
  13500. }
  13501. +
  13502. +#endif
  13503. diff -Nur linux-4.6.6.orig/kernel/panic.c linux-4.6.6/kernel/panic.c
  13504. --- linux-4.6.6.orig/kernel/panic.c 2016-08-10 12:54:57.000000000 +0200
  13505. +++ linux-4.6.6/kernel/panic.c 2016-08-22 21:27:47.389891577 +0200
  13506. @@ -444,9 +444,11 @@
  13507. static int init_oops_id(void)
  13508. {
  13509. +#ifndef CONFIG_PREEMPT_RT_FULL
  13510. if (!oops_id)
  13511. get_random_bytes(&oops_id, sizeof(oops_id));
  13512. else
  13513. +#endif
  13514. oops_id++;
  13515. return 0;
  13516. diff -Nur linux-4.6.6.orig/kernel/power/hibernate.c linux-4.6.6/kernel/power/hibernate.c
  13517. --- linux-4.6.6.orig/kernel/power/hibernate.c 2016-08-10 12:54:57.000000000 +0200
  13518. +++ linux-4.6.6/kernel/power/hibernate.c 2016-08-22 21:27:47.389891577 +0200
  13519. @@ -285,6 +285,8 @@
  13520. local_irq_disable();
  13521. + system_state = SYSTEM_SUSPEND;
  13522. +
  13523. error = syscore_suspend();
  13524. if (error) {
  13525. printk(KERN_ERR "PM: Some system devices failed to power down, "
  13526. @@ -314,6 +316,7 @@
  13527. syscore_resume();
  13528. Enable_irqs:
  13529. + system_state = SYSTEM_RUNNING;
  13530. local_irq_enable();
  13531. Enable_cpus:
  13532. @@ -438,6 +441,7 @@
  13533. goto Enable_cpus;
  13534. local_irq_disable();
  13535. + system_state = SYSTEM_SUSPEND;
  13536. error = syscore_suspend();
  13537. if (error)
  13538. @@ -471,6 +475,7 @@
  13539. syscore_resume();
  13540. Enable_irqs:
  13541. + system_state = SYSTEM_RUNNING;
  13542. local_irq_enable();
  13543. Enable_cpus:
  13544. @@ -556,6 +561,7 @@
  13545. goto Enable_cpus;
  13546. local_irq_disable();
  13547. + system_state = SYSTEM_SUSPEND;
  13548. syscore_suspend();
  13549. if (pm_wakeup_pending()) {
  13550. error = -EAGAIN;
  13551. @@ -568,6 +574,7 @@
  13552. Power_up:
  13553. syscore_resume();
  13554. + system_state = SYSTEM_RUNNING;
  13555. local_irq_enable();
  13556. Enable_cpus:
  13557. @@ -642,6 +649,10 @@
  13558. cpu_relax();
  13559. }
  13560. +#ifndef CONFIG_SUSPEND
  13561. +bool pm_in_action;
  13562. +#endif
  13563. +
  13564. /**
  13565. * hibernate - Carry out system hibernation, including saving the image.
  13566. */
  13567. @@ -654,6 +665,8 @@
  13568. return -EPERM;
  13569. }
  13570. + pm_in_action = true;
  13571. +
  13572. lock_system_sleep();
  13573. /* The snapshot device should not be opened while we're running */
  13574. if (!atomic_add_unless(&snapshot_device_available, -1, 0)) {
  13575. @@ -719,6 +732,7 @@
  13576. atomic_inc(&snapshot_device_available);
  13577. Unlock:
  13578. unlock_system_sleep();
  13579. + pm_in_action = false;
  13580. return error;
  13581. }
  13582. diff -Nur linux-4.6.6.orig/kernel/power/suspend.c linux-4.6.6/kernel/power/suspend.c
  13583. --- linux-4.6.6.orig/kernel/power/suspend.c 2016-08-10 12:54:57.000000000 +0200
  13584. +++ linux-4.6.6/kernel/power/suspend.c 2016-08-22 21:27:47.389891577 +0200
  13585. @@ -359,6 +359,8 @@
  13586. arch_suspend_disable_irqs();
  13587. BUG_ON(!irqs_disabled());
  13588. + system_state = SYSTEM_SUSPEND;
  13589. +
  13590. error = syscore_suspend();
  13591. if (!error) {
  13592. *wakeup = pm_wakeup_pending();
  13593. @@ -375,6 +377,8 @@
  13594. syscore_resume();
  13595. }
  13596. + system_state = SYSTEM_RUNNING;
  13597. +
  13598. arch_suspend_enable_irqs();
  13599. BUG_ON(irqs_disabled());
  13600. @@ -517,6 +521,8 @@
  13601. return error;
  13602. }
  13603. +bool pm_in_action;
  13604. +
  13605. /**
  13606. * pm_suspend - Externally visible function for suspending the system.
  13607. * @state: System sleep state to enter.
  13608. @@ -531,6 +537,8 @@
  13609. if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX)
  13610. return -EINVAL;
  13611. + pm_in_action = true;
  13612. +
  13613. error = enter_state(state);
  13614. if (error) {
  13615. suspend_stats.fail++;
  13616. @@ -538,6 +546,7 @@
  13617. } else {
  13618. suspend_stats.success++;
  13619. }
  13620. + pm_in_action = false;
  13621. return error;
  13622. }
  13623. EXPORT_SYMBOL(pm_suspend);
  13624. diff -Nur linux-4.6.6.orig/kernel/printk/printk.c linux-4.6.6/kernel/printk/printk.c
  13625. --- linux-4.6.6.orig/kernel/printk/printk.c 2016-08-10 12:54:57.000000000 +0200
  13626. +++ linux-4.6.6/kernel/printk/printk.c 2016-08-22 21:27:47.393891830 +0200
  13627. @@ -246,6 +246,65 @@
  13628. */
  13629. static DEFINE_RAW_SPINLOCK(logbuf_lock);
  13630. +#ifdef CONFIG_EARLY_PRINTK
  13631. +struct console *early_console;
  13632. +
  13633. +static void early_vprintk(const char *fmt, va_list ap)
  13634. +{
  13635. + if (early_console) {
  13636. + char buf[512];
  13637. + int n = vscnprintf(buf, sizeof(buf), fmt, ap);
  13638. +
  13639. + early_console->write(early_console, buf, n);
  13640. + }
  13641. +}
  13642. +
  13643. +asmlinkage void early_printk(const char *fmt, ...)
  13644. +{
  13645. + va_list ap;
  13646. +
  13647. + va_start(ap, fmt);
  13648. + early_vprintk(fmt, ap);
  13649. + va_end(ap);
  13650. +}
  13651. +
  13652. +/*
  13653. + * This is independent of any log levels - a global
  13654. + * kill switch that turns off all of printk.
  13655. + *
  13656. + * Used by the NMI watchdog if early-printk is enabled.
  13657. + */
  13658. +static bool __read_mostly printk_killswitch;
  13659. +
  13660. +static int __init force_early_printk_setup(char *str)
  13661. +{
  13662. + printk_killswitch = true;
  13663. + return 0;
  13664. +}
  13665. +early_param("force_early_printk", force_early_printk_setup);
  13666. +
  13667. +void printk_kill(void)
  13668. +{
  13669. + printk_killswitch = true;
  13670. +}
  13671. +
  13672. +#ifdef CONFIG_PRINTK
  13673. +static int forced_early_printk(const char *fmt, va_list ap)
  13674. +{
  13675. + if (!printk_killswitch)
  13676. + return 0;
  13677. + early_vprintk(fmt, ap);
  13678. + return 1;
  13679. +}
  13680. +#endif
  13681. +
  13682. +#else
  13683. +static inline int forced_early_printk(const char *fmt, va_list ap)
  13684. +{
  13685. + return 0;
  13686. +}
  13687. +#endif
  13688. +
  13689. #ifdef CONFIG_PRINTK
  13690. DECLARE_WAIT_QUEUE_HEAD(log_wait);
  13691. /* the next printk record to read by syslog(READ) or /proc/kmsg */
  13692. @@ -1209,6 +1268,7 @@
  13693. {
  13694. char *text;
  13695. int len = 0;
  13696. + int attempts = 0;
  13697. text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
  13698. if (!text)
  13699. @@ -1220,6 +1280,14 @@
  13700. u64 seq;
  13701. u32 idx;
  13702. enum log_flags prev;
  13703. + int num_msg;
  13704. +try_again:
  13705. + attempts++;
  13706. + if (attempts > 10) {
  13707. + len = -EBUSY;
  13708. + goto out;
  13709. + }
  13710. + num_msg = 0;
  13711. /*
  13712. * Find first record that fits, including all following records,
  13713. @@ -1235,6 +1303,14 @@
  13714. prev = msg->flags;
  13715. idx = log_next(idx);
  13716. seq++;
  13717. + num_msg++;
  13718. + if (num_msg > 5) {
  13719. + num_msg = 0;
  13720. + raw_spin_unlock_irq(&logbuf_lock);
  13721. + raw_spin_lock_irq(&logbuf_lock);
  13722. + if (clear_seq < log_first_seq)
  13723. + goto try_again;
  13724. + }
  13725. }
  13726. /* move first record forward until length fits into the buffer */
  13727. @@ -1248,6 +1324,14 @@
  13728. prev = msg->flags;
  13729. idx = log_next(idx);
  13730. seq++;
  13731. + num_msg++;
  13732. + if (num_msg > 5) {
  13733. + num_msg = 0;
  13734. + raw_spin_unlock_irq(&logbuf_lock);
  13735. + raw_spin_lock_irq(&logbuf_lock);
  13736. + if (clear_seq < log_first_seq)
  13737. + goto try_again;
  13738. + }
  13739. }
  13740. /* last message fitting into this dump */
  13741. @@ -1288,6 +1372,7 @@
  13742. clear_seq = log_next_seq;
  13743. clear_idx = log_next_idx;
  13744. }
  13745. +out:
  13746. raw_spin_unlock_irq(&logbuf_lock);
  13747. kfree(text);
  13748. @@ -1443,6 +1528,12 @@
  13749. if (!console_drivers)
  13750. return;
  13751. + if (IS_ENABLED(CONFIG_PREEMPT_RT_BASE)) {
  13752. + if (in_irq() || in_nmi())
  13753. + return;
  13754. + }
  13755. +
  13756. + migrate_disable();
  13757. for_each_console(con) {
  13758. if (exclusive_console && con != exclusive_console)
  13759. continue;
  13760. @@ -1458,6 +1549,7 @@
  13761. else
  13762. con->write(con, text, len);
  13763. }
  13764. + migrate_enable();
  13765. }
  13766. /*
  13767. @@ -1620,6 +1712,13 @@
  13768. /* cpu currently holding logbuf_lock in this function */
  13769. static unsigned int logbuf_cpu = UINT_MAX;
  13770. + /*
  13771. + * Fall back to early_printk if a debugging subsystem has
  13772. + * killed printk output
  13773. + */
  13774. + if (unlikely(forced_early_printk(fmt, args)))
  13775. + return 1;
  13776. +
  13777. if (level == LOGLEVEL_SCHED) {
  13778. level = LOGLEVEL_DEFAULT;
  13779. in_sched = true;
  13780. @@ -1755,13 +1854,23 @@
  13781. /* If called from the scheduler, we can not call up(). */
  13782. if (!in_sched) {
  13783. + int may_trylock = 1;
  13784. +
  13785. lockdep_off();
  13786. +#ifdef CONFIG_PREEMPT_RT_FULL
  13787. + /*
  13788. + * we can't take a sleeping lock with IRQs or preeption disabled
  13789. + * so we can't print in these contexts
  13790. + */
  13791. + if (!(preempt_count() == 0 && !irqs_disabled()))
  13792. + may_trylock = 0;
  13793. +#endif
  13794. /*
  13795. * Try to acquire and then immediately release the console
  13796. * semaphore. The release will print out buffers and wake up
  13797. * /dev/kmsg and syslog() users.
  13798. */
  13799. - if (console_trylock())
  13800. + if (may_trylock && console_trylock())
  13801. console_unlock();
  13802. lockdep_on();
  13803. }
  13804. @@ -1901,26 +2010,6 @@
  13805. #endif /* CONFIG_PRINTK */
  13806. -#ifdef CONFIG_EARLY_PRINTK
  13807. -struct console *early_console;
  13808. -
  13809. -asmlinkage __visible void early_printk(const char *fmt, ...)
  13810. -{
  13811. - va_list ap;
  13812. - char buf[512];
  13813. - int n;
  13814. -
  13815. - if (!early_console)
  13816. - return;
  13817. -
  13818. - va_start(ap, fmt);
  13819. - n = vscnprintf(buf, sizeof(buf), fmt, ap);
  13820. - va_end(ap);
  13821. -
  13822. - early_console->write(early_console, buf, n);
  13823. -}
  13824. -#endif
  13825. -
  13826. static int __add_preferred_console(char *name, int idx, char *options,
  13827. char *brl_options)
  13828. {
  13829. @@ -2183,11 +2272,16 @@
  13830. goto out;
  13831. len = cont_print_text(text, size);
  13832. +#ifdef CONFIG_PREEMPT_RT_FULL
  13833. + raw_spin_unlock_irqrestore(&logbuf_lock, flags);
  13834. + call_console_drivers(cont.level, NULL, 0, text, len);
  13835. +#else
  13836. raw_spin_unlock(&logbuf_lock);
  13837. stop_critical_timings();
  13838. call_console_drivers(cont.level, NULL, 0, text, len);
  13839. start_critical_timings();
  13840. local_irq_restore(flags);
  13841. +#endif
  13842. return;
  13843. out:
  13844. raw_spin_unlock_irqrestore(&logbuf_lock, flags);
  13845. @@ -2309,13 +2403,17 @@
  13846. console_idx = log_next(console_idx);
  13847. console_seq++;
  13848. console_prev = msg->flags;
  13849. +#ifdef CONFIG_PREEMPT_RT_FULL
  13850. + raw_spin_unlock_irqrestore(&logbuf_lock, flags);
  13851. + call_console_drivers(level, ext_text, ext_len, text, len);
  13852. +#else
  13853. raw_spin_unlock(&logbuf_lock);
  13854. stop_critical_timings(); /* don't trace print latency */
  13855. call_console_drivers(level, ext_text, ext_len, text, len);
  13856. start_critical_timings();
  13857. local_irq_restore(flags);
  13858. -
  13859. +#endif
  13860. if (do_cond_resched)
  13861. cond_resched();
  13862. }
  13863. @@ -2367,6 +2465,11 @@
  13864. {
  13865. struct console *c;
  13866. + if (IS_ENABLED(CONFIG_PREEMPT_RT_BASE)) {
  13867. + if (in_irq() || in_nmi())
  13868. + return;
  13869. + }
  13870. +
  13871. /*
  13872. * console_unblank can no longer be called in interrupt context unless
  13873. * oops_in_progress is set to 1..
  13874. diff -Nur linux-4.6.6.orig/kernel/ptrace.c linux-4.6.6/kernel/ptrace.c
  13875. --- linux-4.6.6.orig/kernel/ptrace.c 2016-08-10 12:54:57.000000000 +0200
  13876. +++ linux-4.6.6/kernel/ptrace.c 2016-08-22 21:27:47.393891830 +0200
  13877. @@ -128,7 +128,14 @@
  13878. spin_lock_irq(&task->sighand->siglock);
  13879. if (task_is_traced(task) && !__fatal_signal_pending(task)) {
  13880. - task->state = __TASK_TRACED;
  13881. + unsigned long flags;
  13882. +
  13883. + raw_spin_lock_irqsave(&task->pi_lock, flags);
  13884. + if (task->state & __TASK_TRACED)
  13885. + task->state = __TASK_TRACED;
  13886. + else
  13887. + task->saved_state = __TASK_TRACED;
  13888. + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  13889. ret = true;
  13890. }
  13891. spin_unlock_irq(&task->sighand->siglock);
  13892. diff -Nur linux-4.6.6.orig/kernel/rcu/rcutorture.c linux-4.6.6/kernel/rcu/rcutorture.c
  13893. --- linux-4.6.6.orig/kernel/rcu/rcutorture.c 2016-08-10 12:54:57.000000000 +0200
  13894. +++ linux-4.6.6/kernel/rcu/rcutorture.c 2016-08-22 21:27:47.393891830 +0200
  13895. @@ -409,6 +409,7 @@
  13896. .name = "rcu"
  13897. };
  13898. +#ifndef CONFIG_PREEMPT_RT_FULL
  13899. /*
  13900. * Definitions for rcu_bh torture testing.
  13901. */
  13902. @@ -448,6 +449,12 @@
  13903. .name = "rcu_bh"
  13904. };
  13905. +#else
  13906. +static struct rcu_torture_ops rcu_bh_ops = {
  13907. + .ttype = INVALID_RCU_FLAVOR,
  13908. +};
  13909. +#endif
  13910. +
  13911. /*
  13912. * Don't even think about trying any of these in real life!!!
  13913. * The names includes "busted", and they really means it!
  13914. diff -Nur linux-4.6.6.orig/kernel/rcu/tree.c linux-4.6.6/kernel/rcu/tree.c
  13915. --- linux-4.6.6.orig/kernel/rcu/tree.c 2016-08-10 12:54:57.000000000 +0200
  13916. +++ linux-4.6.6/kernel/rcu/tree.c 2016-08-22 21:27:47.397892090 +0200
  13917. @@ -56,6 +56,11 @@
  13918. #include <linux/random.h>
  13919. #include <linux/trace_events.h>
  13920. #include <linux/suspend.h>
  13921. +#include <linux/delay.h>
  13922. +#include <linux/gfp.h>
  13923. +#include <linux/oom.h>
  13924. +#include <linux/smpboot.h>
  13925. +#include "../time/tick-internal.h"
  13926. #include "tree.h"
  13927. #include "rcu.h"
  13928. @@ -254,6 +259,19 @@
  13929. this_cpu_ptr(&rcu_sched_data), true);
  13930. }
  13931. +#ifdef CONFIG_PREEMPT_RT_FULL
  13932. +static void rcu_preempt_qs(void);
  13933. +
  13934. +void rcu_bh_qs(void)
  13935. +{
  13936. + unsigned long flags;
  13937. +
  13938. + /* Callers to this function, rcu_preempt_qs(), must disable irqs. */
  13939. + local_irq_save(flags);
  13940. + rcu_preempt_qs();
  13941. + local_irq_restore(flags);
  13942. +}
  13943. +#else
  13944. void rcu_bh_qs(void)
  13945. {
  13946. if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
  13947. @@ -263,6 +281,7 @@
  13948. __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
  13949. }
  13950. }
  13951. +#endif
  13952. static DEFINE_PER_CPU(int, rcu_sched_qs_mask);
  13953. @@ -426,11 +445,13 @@
  13954. /*
  13955. * Return the number of RCU BH batches started thus far for debug & stats.
  13956. */
  13957. +#ifndef CONFIG_PREEMPT_RT_FULL
  13958. unsigned long rcu_batches_started_bh(void)
  13959. {
  13960. return rcu_bh_state.gpnum;
  13961. }
  13962. EXPORT_SYMBOL_GPL(rcu_batches_started_bh);
  13963. +#endif
  13964. /*
  13965. * Return the number of RCU batches completed thus far for debug & stats.
  13966. @@ -450,6 +471,7 @@
  13967. }
  13968. EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
  13969. +#ifndef CONFIG_PREEMPT_RT_FULL
  13970. /*
  13971. * Return the number of RCU BH batches completed thus far for debug & stats.
  13972. */
  13973. @@ -477,6 +499,13 @@
  13974. }
  13975. EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
  13976. +#else
  13977. +void rcu_force_quiescent_state(void)
  13978. +{
  13979. +}
  13980. +EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
  13981. +#endif
  13982. +
  13983. /*
  13984. * Force a quiescent state for RCU-sched.
  13985. */
  13986. @@ -527,9 +556,11 @@
  13987. case RCU_FLAVOR:
  13988. rsp = rcu_state_p;
  13989. break;
  13990. +#ifndef CONFIG_PREEMPT_RT_FULL
  13991. case RCU_BH_FLAVOR:
  13992. rsp = &rcu_bh_state;
  13993. break;
  13994. +#endif
  13995. case RCU_SCHED_FLAVOR:
  13996. rsp = &rcu_sched_state;
  13997. break;
  13998. @@ -2920,18 +2951,17 @@
  13999. /*
  14000. * Do RCU core processing for the current CPU.
  14001. */
  14002. -static void rcu_process_callbacks(struct softirq_action *unused)
  14003. +static void rcu_process_callbacks(void)
  14004. {
  14005. struct rcu_state *rsp;
  14006. if (cpu_is_offline(smp_processor_id()))
  14007. return;
  14008. - trace_rcu_utilization(TPS("Start RCU core"));
  14009. for_each_rcu_flavor(rsp)
  14010. __rcu_process_callbacks(rsp);
  14011. - trace_rcu_utilization(TPS("End RCU core"));
  14012. }
  14013. +static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
  14014. /*
  14015. * Schedule RCU callback invocation. If the specified type of RCU
  14016. * does not support RCU priority boosting, just do a direct call,
  14017. @@ -2943,18 +2973,105 @@
  14018. {
  14019. if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
  14020. return;
  14021. - if (likely(!rsp->boost)) {
  14022. - rcu_do_batch(rsp, rdp);
  14023. + rcu_do_batch(rsp, rdp);
  14024. +}
  14025. +
  14026. +static void rcu_wake_cond(struct task_struct *t, int status)
  14027. +{
  14028. + /*
  14029. + * If the thread is yielding, only wake it when this
  14030. + * is invoked from idle
  14031. + */
  14032. + if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
  14033. + wake_up_process(t);
  14034. +}
  14035. +
  14036. +/*
  14037. + * Wake up this CPU's rcuc kthread to do RCU core processing.
  14038. + */
  14039. +static void invoke_rcu_core(void)
  14040. +{
  14041. + unsigned long flags;
  14042. + struct task_struct *t;
  14043. +
  14044. + if (!cpu_online(smp_processor_id()))
  14045. return;
  14046. + local_irq_save(flags);
  14047. + __this_cpu_write(rcu_cpu_has_work, 1);
  14048. + t = __this_cpu_read(rcu_cpu_kthread_task);
  14049. + if (t != NULL && current != t)
  14050. + rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status));
  14051. + local_irq_restore(flags);
  14052. +}
  14053. +
  14054. +static void rcu_cpu_kthread_park(unsigned int cpu)
  14055. +{
  14056. + per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
  14057. +}
  14058. +
  14059. +static int rcu_cpu_kthread_should_run(unsigned int cpu)
  14060. +{
  14061. + return __this_cpu_read(rcu_cpu_has_work);
  14062. +}
  14063. +
  14064. +/*
  14065. + * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
  14066. + * RCU softirq used in flavors and configurations of RCU that do not
  14067. + * support RCU priority boosting.
  14068. + */
  14069. +static void rcu_cpu_kthread(unsigned int cpu)
  14070. +{
  14071. + unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
  14072. + char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
  14073. + int spincnt;
  14074. +
  14075. + for (spincnt = 0; spincnt < 10; spincnt++) {
  14076. + trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
  14077. + local_bh_disable();
  14078. + *statusp = RCU_KTHREAD_RUNNING;
  14079. + this_cpu_inc(rcu_cpu_kthread_loops);
  14080. + local_irq_disable();
  14081. + work = *workp;
  14082. + *workp = 0;
  14083. + local_irq_enable();
  14084. + if (work)
  14085. + rcu_process_callbacks();
  14086. + local_bh_enable();
  14087. + if (*workp == 0) {
  14088. + trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
  14089. + *statusp = RCU_KTHREAD_WAITING;
  14090. + return;
  14091. + }
  14092. }
  14093. - invoke_rcu_callbacks_kthread();
  14094. + *statusp = RCU_KTHREAD_YIELDING;
  14095. + trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
  14096. + schedule_timeout_interruptible(2);
  14097. + trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
  14098. + *statusp = RCU_KTHREAD_WAITING;
  14099. }
  14100. -static void invoke_rcu_core(void)
  14101. +static struct smp_hotplug_thread rcu_cpu_thread_spec = {
  14102. + .store = &rcu_cpu_kthread_task,
  14103. + .thread_should_run = rcu_cpu_kthread_should_run,
  14104. + .thread_fn = rcu_cpu_kthread,
  14105. + .thread_comm = "rcuc/%u",
  14106. + .setup = rcu_cpu_kthread_setup,
  14107. + .park = rcu_cpu_kthread_park,
  14108. +};
  14109. +
  14110. +/*
  14111. + * Spawn per-CPU RCU core processing kthreads.
  14112. + */
  14113. +static int __init rcu_spawn_core_kthreads(void)
  14114. {
  14115. - if (cpu_online(smp_processor_id()))
  14116. - raise_softirq(RCU_SOFTIRQ);
  14117. + int cpu;
  14118. +
  14119. + for_each_possible_cpu(cpu)
  14120. + per_cpu(rcu_cpu_has_work, cpu) = 0;
  14121. + BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
  14122. + return 0;
  14123. }
  14124. +early_initcall(rcu_spawn_core_kthreads);
  14125. /*
  14126. * Handle any core-RCU processing required by a call_rcu() invocation.
  14127. @@ -3099,6 +3216,7 @@
  14128. }
  14129. EXPORT_SYMBOL_GPL(call_rcu_sched);
  14130. +#ifndef CONFIG_PREEMPT_RT_FULL
  14131. /*
  14132. * Queue an RCU callback for invocation after a quicker grace period.
  14133. */
  14134. @@ -3107,6 +3225,7 @@
  14135. __call_rcu(head, func, &rcu_bh_state, -1, 0);
  14136. }
  14137. EXPORT_SYMBOL_GPL(call_rcu_bh);
  14138. +#endif
  14139. /*
  14140. * Queue an RCU callback for lazy invocation after a grace period.
  14141. @@ -3198,6 +3317,7 @@
  14142. }
  14143. EXPORT_SYMBOL_GPL(synchronize_sched);
  14144. +#ifndef CONFIG_PREEMPT_RT_FULL
  14145. /**
  14146. * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
  14147. *
  14148. @@ -3224,6 +3344,7 @@
  14149. wait_rcu_gp(call_rcu_bh);
  14150. }
  14151. EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
  14152. +#endif
  14153. /**
  14154. * get_state_synchronize_rcu - Snapshot current RCU state
  14155. @@ -4104,6 +4225,7 @@
  14156. mutex_unlock(&rsp->barrier_mutex);
  14157. }
  14158. +#ifndef CONFIG_PREEMPT_RT_FULL
  14159. /**
  14160. * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
  14161. */
  14162. @@ -4112,6 +4234,7 @@
  14163. _rcu_barrier(&rcu_bh_state);
  14164. }
  14165. EXPORT_SYMBOL_GPL(rcu_barrier_bh);
  14166. +#endif
  14167. /**
  14168. * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
  14169. @@ -4609,12 +4732,13 @@
  14170. rcu_bootup_announce();
  14171. rcu_init_geometry();
  14172. +#ifndef CONFIG_PREEMPT_RT_FULL
  14173. rcu_init_one(&rcu_bh_state);
  14174. +#endif
  14175. rcu_init_one(&rcu_sched_state);
  14176. if (dump_tree)
  14177. rcu_dump_rcu_node_tree(&rcu_sched_state);
  14178. __rcu_init_preempt();
  14179. - open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
  14180. /*
  14181. * We don't need protection against CPU-hotplug here because
  14182. diff -Nur linux-4.6.6.orig/kernel/rcu/tree.h linux-4.6.6/kernel/rcu/tree.h
  14183. --- linux-4.6.6.orig/kernel/rcu/tree.h 2016-08-10 12:54:57.000000000 +0200
  14184. +++ linux-4.6.6/kernel/rcu/tree.h 2016-08-22 21:27:47.397892090 +0200
  14185. @@ -572,18 +572,18 @@
  14186. */
  14187. extern struct rcu_state rcu_sched_state;
  14188. +#ifndef CONFIG_PREEMPT_RT_FULL
  14189. extern struct rcu_state rcu_bh_state;
  14190. +#endif
  14191. #ifdef CONFIG_PREEMPT_RCU
  14192. extern struct rcu_state rcu_preempt_state;
  14193. #endif /* #ifdef CONFIG_PREEMPT_RCU */
  14194. -#ifdef CONFIG_RCU_BOOST
  14195. DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
  14196. DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
  14197. DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
  14198. DECLARE_PER_CPU(char, rcu_cpu_has_work);
  14199. -#endif /* #ifdef CONFIG_RCU_BOOST */
  14200. #ifndef RCU_TREE_NONCORE
  14201. @@ -603,10 +603,9 @@
  14202. static void __init __rcu_init_preempt(void);
  14203. static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
  14204. static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
  14205. -static void invoke_rcu_callbacks_kthread(void);
  14206. static bool rcu_is_callbacks_kthread(void);
  14207. +static void rcu_cpu_kthread_setup(unsigned int cpu);
  14208. #ifdef CONFIG_RCU_BOOST
  14209. -static void rcu_preempt_do_callbacks(void);
  14210. static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
  14211. struct rcu_node *rnp);
  14212. #endif /* #ifdef CONFIG_RCU_BOOST */
  14213. diff -Nur linux-4.6.6.orig/kernel/rcu/tree_plugin.h linux-4.6.6/kernel/rcu/tree_plugin.h
  14214. --- linux-4.6.6.orig/kernel/rcu/tree_plugin.h 2016-08-10 12:54:57.000000000 +0200
  14215. +++ linux-4.6.6/kernel/rcu/tree_plugin.h 2016-08-22 21:27:47.401892343 +0200
  14216. @@ -24,25 +24,10 @@
  14217. * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  14218. */
  14219. -#include <linux/delay.h>
  14220. -#include <linux/gfp.h>
  14221. -#include <linux/oom.h>
  14222. -#include <linux/smpboot.h>
  14223. -#include "../time/tick-internal.h"
  14224. -
  14225. #ifdef CONFIG_RCU_BOOST
  14226. #include "../locking/rtmutex_common.h"
  14227. -/*
  14228. - * Control variables for per-CPU and per-rcu_node kthreads. These
  14229. - * handle all flavors of RCU.
  14230. - */
  14231. -static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
  14232. -DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
  14233. -DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
  14234. -DEFINE_PER_CPU(char, rcu_cpu_has_work);
  14235. -
  14236. #else /* #ifdef CONFIG_RCU_BOOST */
  14237. /*
  14238. @@ -55,6 +40,14 @@
  14239. #endif /* #else #ifdef CONFIG_RCU_BOOST */
  14240. +/*
  14241. + * Control variables for per-CPU and per-rcu_node kthreads. These
  14242. + * handle all flavors of RCU.
  14243. + */
  14244. +DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
  14245. +DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
  14246. +DEFINE_PER_CPU(char, rcu_cpu_has_work);
  14247. +
  14248. #ifdef CONFIG_RCU_NOCB_CPU
  14249. static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
  14250. static bool have_rcu_nocb_mask; /* Was rcu_nocb_mask allocated? */
  14251. @@ -428,7 +421,7 @@
  14252. }
  14253. /* Hardware IRQ handlers cannot block, complain if they get here. */
  14254. - if (in_irq() || in_serving_softirq()) {
  14255. + if (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET)) {
  14256. lockdep_rcu_suspicious(__FILE__, __LINE__,
  14257. "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
  14258. pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
  14259. @@ -634,15 +627,6 @@
  14260. t->rcu_read_unlock_special.b.need_qs = true;
  14261. }
  14262. -#ifdef CONFIG_RCU_BOOST
  14263. -
  14264. -static void rcu_preempt_do_callbacks(void)
  14265. -{
  14266. - rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
  14267. -}
  14268. -
  14269. -#endif /* #ifdef CONFIG_RCU_BOOST */
  14270. -
  14271. /*
  14272. * Queue a preemptible-RCU callback for invocation after a grace period.
  14273. */
  14274. @@ -924,6 +908,19 @@
  14275. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  14276. +/*
  14277. + * If boosting, set rcuc kthreads to realtime priority.
  14278. + */
  14279. +static void rcu_cpu_kthread_setup(unsigned int cpu)
  14280. +{
  14281. +#ifdef CONFIG_RCU_BOOST
  14282. + struct sched_param sp;
  14283. +
  14284. + sp.sched_priority = kthread_prio;
  14285. + sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
  14286. +#endif /* #ifdef CONFIG_RCU_BOOST */
  14287. +}
  14288. +
  14289. #ifdef CONFIG_RCU_BOOST
  14290. #include "../locking/rtmutex_common.h"
  14291. @@ -955,16 +952,6 @@
  14292. #endif /* #else #ifdef CONFIG_RCU_TRACE */
  14293. -static void rcu_wake_cond(struct task_struct *t, int status)
  14294. -{
  14295. - /*
  14296. - * If the thread is yielding, only wake it when this
  14297. - * is invoked from idle
  14298. - */
  14299. - if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
  14300. - wake_up_process(t);
  14301. -}
  14302. -
  14303. /*
  14304. * Carry out RCU priority boosting on the task indicated by ->exp_tasks
  14305. * or ->boost_tasks, advancing the pointer to the next task in the
  14306. @@ -1108,23 +1095,6 @@
  14307. }
  14308. /*
  14309. - * Wake up the per-CPU kthread to invoke RCU callbacks.
  14310. - */
  14311. -static void invoke_rcu_callbacks_kthread(void)
  14312. -{
  14313. - unsigned long flags;
  14314. -
  14315. - local_irq_save(flags);
  14316. - __this_cpu_write(rcu_cpu_has_work, 1);
  14317. - if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
  14318. - current != __this_cpu_read(rcu_cpu_kthread_task)) {
  14319. - rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
  14320. - __this_cpu_read(rcu_cpu_kthread_status));
  14321. - }
  14322. - local_irq_restore(flags);
  14323. -}
  14324. -
  14325. -/*
  14326. * Is the current CPU running the RCU-callbacks kthread?
  14327. * Caller must have preemption disabled.
  14328. */
  14329. @@ -1178,67 +1148,6 @@
  14330. return 0;
  14331. }
  14332. -static void rcu_kthread_do_work(void)
  14333. -{
  14334. - rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
  14335. - rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
  14336. - rcu_preempt_do_callbacks();
  14337. -}
  14338. -
  14339. -static void rcu_cpu_kthread_setup(unsigned int cpu)
  14340. -{
  14341. - struct sched_param sp;
  14342. -
  14343. - sp.sched_priority = kthread_prio;
  14344. - sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
  14345. -}
  14346. -
  14347. -static void rcu_cpu_kthread_park(unsigned int cpu)
  14348. -{
  14349. - per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
  14350. -}
  14351. -
  14352. -static int rcu_cpu_kthread_should_run(unsigned int cpu)
  14353. -{
  14354. - return __this_cpu_read(rcu_cpu_has_work);
  14355. -}
  14356. -
  14357. -/*
  14358. - * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
  14359. - * RCU softirq used in flavors and configurations of RCU that do not
  14360. - * support RCU priority boosting.
  14361. - */
  14362. -static void rcu_cpu_kthread(unsigned int cpu)
  14363. -{
  14364. - unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
  14365. - char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
  14366. - int spincnt;
  14367. -
  14368. - for (spincnt = 0; spincnt < 10; spincnt++) {
  14369. - trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
  14370. - local_bh_disable();
  14371. - *statusp = RCU_KTHREAD_RUNNING;
  14372. - this_cpu_inc(rcu_cpu_kthread_loops);
  14373. - local_irq_disable();
  14374. - work = *workp;
  14375. - *workp = 0;
  14376. - local_irq_enable();
  14377. - if (work)
  14378. - rcu_kthread_do_work();
  14379. - local_bh_enable();
  14380. - if (*workp == 0) {
  14381. - trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
  14382. - *statusp = RCU_KTHREAD_WAITING;
  14383. - return;
  14384. - }
  14385. - }
  14386. - *statusp = RCU_KTHREAD_YIELDING;
  14387. - trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
  14388. - schedule_timeout_interruptible(2);
  14389. - trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
  14390. - *statusp = RCU_KTHREAD_WAITING;
  14391. -}
  14392. -
  14393. /*
  14394. * Set the per-rcu_node kthread's affinity to cover all CPUs that are
  14395. * served by the rcu_node in question. The CPU hotplug lock is still
  14396. @@ -1268,26 +1177,12 @@
  14397. free_cpumask_var(cm);
  14398. }
  14399. -static struct smp_hotplug_thread rcu_cpu_thread_spec = {
  14400. - .store = &rcu_cpu_kthread_task,
  14401. - .thread_should_run = rcu_cpu_kthread_should_run,
  14402. - .thread_fn = rcu_cpu_kthread,
  14403. - .thread_comm = "rcuc/%u",
  14404. - .setup = rcu_cpu_kthread_setup,
  14405. - .park = rcu_cpu_kthread_park,
  14406. -};
  14407. -
  14408. /*
  14409. * Spawn boost kthreads -- called as soon as the scheduler is running.
  14410. */
  14411. static void __init rcu_spawn_boost_kthreads(void)
  14412. {
  14413. struct rcu_node *rnp;
  14414. - int cpu;
  14415. -
  14416. - for_each_possible_cpu(cpu)
  14417. - per_cpu(rcu_cpu_has_work, cpu) = 0;
  14418. - BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
  14419. rcu_for_each_leaf_node(rcu_state_p, rnp)
  14420. (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
  14421. }
  14422. @@ -1310,11 +1205,6 @@
  14423. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  14424. }
  14425. -static void invoke_rcu_callbacks_kthread(void)
  14426. -{
  14427. - WARN_ON_ONCE(1);
  14428. -}
  14429. -
  14430. static bool rcu_is_callbacks_kthread(void)
  14431. {
  14432. return false;
  14433. @@ -1338,7 +1228,7 @@
  14434. #endif /* #else #ifdef CONFIG_RCU_BOOST */
  14435. -#if !defined(CONFIG_RCU_FAST_NO_HZ)
  14436. +#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL)
  14437. /*
  14438. * Check to see if any future RCU-related work will need to be done
  14439. @@ -1355,7 +1245,9 @@
  14440. return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
  14441. ? 0 : rcu_cpu_has_callbacks(NULL);
  14442. }
  14443. +#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */
  14444. +#if !defined(CONFIG_RCU_FAST_NO_HZ)
  14445. /*
  14446. * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
  14447. * after it.
  14448. @@ -1451,6 +1343,8 @@
  14449. return cbs_ready;
  14450. }
  14451. +#ifndef CONFIG_PREEMPT_RT_FULL
  14452. +
  14453. /*
  14454. * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
  14455. * to invoke. If the CPU has callbacks, try to advance them. Tell the
  14456. @@ -1496,6 +1390,7 @@
  14457. *nextevt = basemono + dj * TICK_NSEC;
  14458. return 0;
  14459. }
  14460. +#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */
  14461. /*
  14462. * Prepare a CPU for idle from an RCU perspective. The first major task
  14463. diff -Nur linux-4.6.6.orig/kernel/rcu/update.c linux-4.6.6/kernel/rcu/update.c
  14464. --- linux-4.6.6.orig/kernel/rcu/update.c 2016-08-10 12:54:57.000000000 +0200
  14465. +++ linux-4.6.6/kernel/rcu/update.c 2016-08-22 21:27:47.401892343 +0200
  14466. @@ -295,6 +295,7 @@
  14467. }
  14468. EXPORT_SYMBOL_GPL(rcu_read_lock_held);
  14469. +#ifndef CONFIG_PREEMPT_RT_FULL
  14470. /**
  14471. * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
  14472. *
  14473. @@ -321,6 +322,7 @@
  14474. return in_softirq() || irqs_disabled();
  14475. }
  14476. EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
  14477. +#endif
  14478. #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
  14479. diff -Nur linux-4.6.6.orig/kernel/relay.c linux-4.6.6/kernel/relay.c
  14480. --- linux-4.6.6.orig/kernel/relay.c 2016-08-10 12:54:57.000000000 +0200
  14481. +++ linux-4.6.6/kernel/relay.c 2016-08-22 21:27:47.405892600 +0200
  14482. @@ -336,6 +336,10 @@
  14483. {
  14484. struct rchan_buf *buf = (struct rchan_buf *)data;
  14485. wake_up_interruptible(&buf->read_wait);
  14486. + /*
  14487. + * Stupid polling for now:
  14488. + */
  14489. + mod_timer(&buf->timer, jiffies + 1);
  14490. }
  14491. /**
  14492. @@ -353,6 +357,7 @@
  14493. init_waitqueue_head(&buf->read_wait);
  14494. kref_init(&buf->kref);
  14495. setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf);
  14496. + mod_timer(&buf->timer, jiffies + 1);
  14497. } else
  14498. del_timer_sync(&buf->timer);
  14499. @@ -736,15 +741,6 @@
  14500. else
  14501. buf->early_bytes += buf->chan->subbuf_size -
  14502. buf->padding[old_subbuf];
  14503. - smp_mb();
  14504. - if (waitqueue_active(&buf->read_wait))
  14505. - /*
  14506. - * Calling wake_up_interruptible() from here
  14507. - * will deadlock if we happen to be logging
  14508. - * from the scheduler (trying to re-grab
  14509. - * rq->lock), so defer it.
  14510. - */
  14511. - mod_timer(&buf->timer, jiffies + 1);
  14512. }
  14513. old = buf->data;
  14514. diff -Nur linux-4.6.6.orig/kernel/sched/completion.c linux-4.6.6/kernel/sched/completion.c
  14515. --- linux-4.6.6.orig/kernel/sched/completion.c 2016-08-10 12:54:57.000000000 +0200
  14516. +++ linux-4.6.6/kernel/sched/completion.c 2016-08-22 21:27:47.405892600 +0200
  14517. @@ -30,10 +30,10 @@
  14518. {
  14519. unsigned long flags;
  14520. - spin_lock_irqsave(&x->wait.lock, flags);
  14521. + raw_spin_lock_irqsave(&x->wait.lock, flags);
  14522. x->done++;
  14523. - __wake_up_locked(&x->wait, TASK_NORMAL, 1);
  14524. - spin_unlock_irqrestore(&x->wait.lock, flags);
  14525. + swake_up_locked(&x->wait);
  14526. + raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  14527. }
  14528. EXPORT_SYMBOL(complete);
  14529. @@ -50,10 +50,10 @@
  14530. {
  14531. unsigned long flags;
  14532. - spin_lock_irqsave(&x->wait.lock, flags);
  14533. + raw_spin_lock_irqsave(&x->wait.lock, flags);
  14534. x->done += UINT_MAX/2;
  14535. - __wake_up_locked(&x->wait, TASK_NORMAL, 0);
  14536. - spin_unlock_irqrestore(&x->wait.lock, flags);
  14537. + swake_up_all_locked(&x->wait);
  14538. + raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  14539. }
  14540. EXPORT_SYMBOL(complete_all);
  14541. @@ -62,20 +62,20 @@
  14542. long (*action)(long), long timeout, int state)
  14543. {
  14544. if (!x->done) {
  14545. - DECLARE_WAITQUEUE(wait, current);
  14546. + DECLARE_SWAITQUEUE(wait);
  14547. - __add_wait_queue_tail_exclusive(&x->wait, &wait);
  14548. + __prepare_to_swait(&x->wait, &wait);
  14549. do {
  14550. if (signal_pending_state(state, current)) {
  14551. timeout = -ERESTARTSYS;
  14552. break;
  14553. }
  14554. __set_current_state(state);
  14555. - spin_unlock_irq(&x->wait.lock);
  14556. + raw_spin_unlock_irq(&x->wait.lock);
  14557. timeout = action(timeout);
  14558. - spin_lock_irq(&x->wait.lock);
  14559. + raw_spin_lock_irq(&x->wait.lock);
  14560. } while (!x->done && timeout);
  14561. - __remove_wait_queue(&x->wait, &wait);
  14562. + __finish_swait(&x->wait, &wait);
  14563. if (!x->done)
  14564. return timeout;
  14565. }
  14566. @@ -89,9 +89,9 @@
  14567. {
  14568. might_sleep();
  14569. - spin_lock_irq(&x->wait.lock);
  14570. + raw_spin_lock_irq(&x->wait.lock);
  14571. timeout = do_wait_for_common(x, action, timeout, state);
  14572. - spin_unlock_irq(&x->wait.lock);
  14573. + raw_spin_unlock_irq(&x->wait.lock);
  14574. return timeout;
  14575. }
  14576. @@ -277,12 +277,12 @@
  14577. if (!READ_ONCE(x->done))
  14578. return 0;
  14579. - spin_lock_irqsave(&x->wait.lock, flags);
  14580. + raw_spin_lock_irqsave(&x->wait.lock, flags);
  14581. if (!x->done)
  14582. ret = 0;
  14583. else
  14584. x->done--;
  14585. - spin_unlock_irqrestore(&x->wait.lock, flags);
  14586. + raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  14587. return ret;
  14588. }
  14589. EXPORT_SYMBOL(try_wait_for_completion);
  14590. @@ -311,7 +311,7 @@
  14591. * after it's acquired the lock.
  14592. */
  14593. smp_rmb();
  14594. - spin_unlock_wait(&x->wait.lock);
  14595. + raw_spin_unlock_wait(&x->wait.lock);
  14596. return true;
  14597. }
  14598. EXPORT_SYMBOL(completion_done);
  14599. diff -Nur linux-4.6.6.orig/kernel/sched/core.c linux-4.6.6/kernel/sched/core.c
  14600. --- linux-4.6.6.orig/kernel/sched/core.c 2016-08-10 12:54:57.000000000 +0200
  14601. +++ linux-4.6.6/kernel/sched/core.c 2016-08-22 21:27:47.413893109 +0200
  14602. @@ -128,7 +128,11 @@
  14603. * Number of tasks to iterate in a single balance run.
  14604. * Limited because this is done with IRQs disabled.
  14605. */
  14606. +#ifndef CONFIG_PREEMPT_RT_FULL
  14607. const_debug unsigned int sysctl_sched_nr_migrate = 32;
  14608. +#else
  14609. +const_debug unsigned int sysctl_sched_nr_migrate = 8;
  14610. +#endif
  14611. /*
  14612. * period over which we average the RT time consumption, measured
  14613. @@ -306,6 +310,7 @@
  14614. hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  14615. rq->hrtick_timer.function = hrtick;
  14616. + rq->hrtick_timer.irqsafe = 1;
  14617. }
  14618. #else /* CONFIG_SCHED_HRTICK */
  14619. static inline void hrtick_clear(struct rq *rq)
  14620. @@ -414,7 +419,7 @@
  14621. head->lastp = &node->next;
  14622. }
  14623. -void wake_up_q(struct wake_q_head *head)
  14624. +void __wake_up_q(struct wake_q_head *head, bool sleeper)
  14625. {
  14626. struct wake_q_node *node = head->first;
  14627. @@ -431,7 +436,10 @@
  14628. * wake_up_process() implies a wmb() to pair with the queueing
  14629. * in wake_q_add() so as not to miss wakeups.
  14630. */
  14631. - wake_up_process(task);
  14632. + if (sleeper)
  14633. + wake_up_lock_sleeper(task);
  14634. + else
  14635. + wake_up_process(task);
  14636. put_task_struct(task);
  14637. }
  14638. }
  14639. @@ -467,6 +475,38 @@
  14640. trace_sched_wake_idle_without_ipi(cpu);
  14641. }
  14642. +#ifdef CONFIG_PREEMPT_LAZY
  14643. +void resched_curr_lazy(struct rq *rq)
  14644. +{
  14645. + struct task_struct *curr = rq->curr;
  14646. + int cpu;
  14647. +
  14648. + if (!sched_feat(PREEMPT_LAZY)) {
  14649. + resched_curr(rq);
  14650. + return;
  14651. + }
  14652. +
  14653. + lockdep_assert_held(&rq->lock);
  14654. +
  14655. + if (test_tsk_need_resched(curr))
  14656. + return;
  14657. +
  14658. + if (test_tsk_need_resched_lazy(curr))
  14659. + return;
  14660. +
  14661. + set_tsk_need_resched_lazy(curr);
  14662. +
  14663. + cpu = cpu_of(rq);
  14664. + if (cpu == smp_processor_id())
  14665. + return;
  14666. +
  14667. + /* NEED_RESCHED_LAZY must be visible before we test polling */
  14668. + smp_mb();
  14669. + if (!tsk_is_polling(curr))
  14670. + smp_send_reschedule(cpu);
  14671. +}
  14672. +#endif
  14673. +
  14674. void resched_cpu(int cpu)
  14675. {
  14676. struct rq *rq = cpu_rq(cpu);
  14677. @@ -490,11 +530,14 @@
  14678. */
  14679. int get_nohz_timer_target(void)
  14680. {
  14681. - int i, cpu = smp_processor_id();
  14682. + int i, cpu;
  14683. struct sched_domain *sd;
  14684. + preempt_disable_rt();
  14685. + cpu = smp_processor_id();
  14686. +
  14687. if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
  14688. - return cpu;
  14689. + goto preempt_en_rt;
  14690. rcu_read_lock();
  14691. for_each_domain(cpu, sd) {
  14692. @@ -510,6 +553,8 @@
  14693. cpu = housekeeping_any_cpu();
  14694. unlock:
  14695. rcu_read_unlock();
  14696. +preempt_en_rt:
  14697. + preempt_enable_rt();
  14698. return cpu;
  14699. }
  14700. /*
  14701. @@ -1051,6 +1096,11 @@
  14702. lockdep_assert_held(&p->pi_lock);
  14703. + if (__migrate_disabled(p)) {
  14704. + cpumask_copy(&p->cpus_allowed, new_mask);
  14705. + return;
  14706. + }
  14707. +
  14708. queued = task_on_rq_queued(p);
  14709. running = task_current(rq, p);
  14710. @@ -1073,6 +1123,84 @@
  14711. enqueue_task(rq, p, ENQUEUE_RESTORE);
  14712. }
  14713. +static DEFINE_PER_CPU(struct cpumask, sched_cpumasks);
  14714. +static DEFINE_MUTEX(sched_down_mutex);
  14715. +static cpumask_t sched_down_cpumask;
  14716. +
  14717. +void tell_sched_cpu_down_begin(int cpu)
  14718. +{
  14719. + mutex_lock(&sched_down_mutex);
  14720. + cpumask_set_cpu(cpu, &sched_down_cpumask);
  14721. + mutex_unlock(&sched_down_mutex);
  14722. +}
  14723. +
  14724. +void tell_sched_cpu_down_done(int cpu)
  14725. +{
  14726. + mutex_lock(&sched_down_mutex);
  14727. + cpumask_clear_cpu(cpu, &sched_down_cpumask);
  14728. + mutex_unlock(&sched_down_mutex);
  14729. +}
  14730. +
  14731. +/**
  14732. + * migrate_me - try to move the current task off this cpu
  14733. + *
  14734. + * Used by the pin_current_cpu() code to try to get tasks
  14735. + * to move off the current CPU as it is going down.
  14736. + * It will only move the task if the task isn't pinned to
  14737. + * the CPU (with migrate_disable, affinity or NO_SETAFFINITY)
  14738. + * and the task has to be in a RUNNING state. Otherwise the
  14739. + * movement of the task will wake it up (change its state
  14740. + * to running) when the task did not expect it.
  14741. + *
  14742. + * Returns 1 if it succeeded in moving the current task
  14743. + * 0 otherwise.
  14744. + */
  14745. +int migrate_me(void)
  14746. +{
  14747. + struct task_struct *p = current;
  14748. + struct migration_arg arg;
  14749. + struct cpumask *cpumask;
  14750. + struct cpumask *mask;
  14751. + unsigned long flags;
  14752. + unsigned int dest_cpu;
  14753. + struct rq *rq;
  14754. +
  14755. + /*
  14756. + * We can not migrate tasks bounded to a CPU or tasks not
  14757. + * running. The movement of the task will wake it up.
  14758. + */
  14759. + if (p->flags & PF_NO_SETAFFINITY || p->state)
  14760. + return 0;
  14761. +
  14762. + mutex_lock(&sched_down_mutex);
  14763. + rq = task_rq_lock(p, &flags);
  14764. +
  14765. + cpumask = this_cpu_ptr(&sched_cpumasks);
  14766. + mask = &p->cpus_allowed;
  14767. +
  14768. + cpumask_andnot(cpumask, mask, &sched_down_cpumask);
  14769. +
  14770. + if (!cpumask_weight(cpumask)) {
  14771. + /* It's only on this CPU? */
  14772. + task_rq_unlock(rq, p, &flags);
  14773. + mutex_unlock(&sched_down_mutex);
  14774. + return 0;
  14775. + }
  14776. +
  14777. + dest_cpu = cpumask_any_and(cpu_active_mask, cpumask);
  14778. +
  14779. + arg.task = p;
  14780. + arg.dest_cpu = dest_cpu;
  14781. +
  14782. + task_rq_unlock(rq, p, &flags);
  14783. +
  14784. + stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
  14785. + tlb_migrate_finish(p->mm);
  14786. + mutex_unlock(&sched_down_mutex);
  14787. +
  14788. + return 1;
  14789. +}
  14790. +
  14791. /*
  14792. * Change a given task's CPU affinity. Migrate the thread to a
  14793. * proper CPU and schedule it away if the CPU it's executing on
  14794. @@ -1112,7 +1240,7 @@
  14795. do_set_cpus_allowed(p, new_mask);
  14796. /* Can the task run on the task's current CPU? If so, we're done */
  14797. - if (cpumask_test_cpu(task_cpu(p), new_mask))
  14798. + if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p))
  14799. goto out;
  14800. dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
  14801. @@ -1299,6 +1427,18 @@
  14802. return ret;
  14803. }
  14804. +static bool check_task_state(struct task_struct *p, long match_state)
  14805. +{
  14806. + bool match = false;
  14807. +
  14808. + raw_spin_lock_irq(&p->pi_lock);
  14809. + if (p->state == match_state || p->saved_state == match_state)
  14810. + match = true;
  14811. + raw_spin_unlock_irq(&p->pi_lock);
  14812. +
  14813. + return match;
  14814. +}
  14815. +
  14816. /*
  14817. * wait_task_inactive - wait for a thread to unschedule.
  14818. *
  14819. @@ -1343,7 +1483,7 @@
  14820. * is actually now running somewhere else!
  14821. */
  14822. while (task_running(rq, p)) {
  14823. - if (match_state && unlikely(p->state != match_state))
  14824. + if (match_state && !check_task_state(p, match_state))
  14825. return 0;
  14826. cpu_relax();
  14827. }
  14828. @@ -1358,7 +1498,8 @@
  14829. running = task_running(rq, p);
  14830. queued = task_on_rq_queued(p);
  14831. ncsw = 0;
  14832. - if (!match_state || p->state == match_state)
  14833. + if (!match_state || p->state == match_state ||
  14834. + p->saved_state == match_state)
  14835. ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
  14836. task_rq_unlock(rq, p, &flags);
  14837. @@ -1515,7 +1656,7 @@
  14838. {
  14839. lockdep_assert_held(&p->pi_lock);
  14840. - if (p->nr_cpus_allowed > 1)
  14841. + if (tsk_nr_cpus_allowed(p) > 1)
  14842. cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
  14843. /*
  14844. @@ -1595,10 +1736,6 @@
  14845. {
  14846. activate_task(rq, p, en_flags);
  14847. p->on_rq = TASK_ON_RQ_QUEUED;
  14848. -
  14849. - /* if a worker is waking up, notify workqueue */
  14850. - if (p->flags & PF_WQ_WORKER)
  14851. - wq_worker_waking_up(p, cpu_of(rq));
  14852. }
  14853. /*
  14854. @@ -1916,8 +2053,27 @@
  14855. */
  14856. smp_mb__before_spinlock();
  14857. raw_spin_lock_irqsave(&p->pi_lock, flags);
  14858. - if (!(p->state & state))
  14859. + if (!(p->state & state)) {
  14860. + /*
  14861. + * The task might be running due to a spinlock sleeper
  14862. + * wakeup. Check the saved state and set it to running
  14863. + * if the wakeup condition is true.
  14864. + */
  14865. + if (!(wake_flags & WF_LOCK_SLEEPER)) {
  14866. + if (p->saved_state & state) {
  14867. + p->saved_state = TASK_RUNNING;
  14868. + success = 1;
  14869. + }
  14870. + }
  14871. goto out;
  14872. + }
  14873. +
  14874. + /*
  14875. + * If this is a regular wakeup, then we can unconditionally
  14876. + * clear the saved state of a "lock sleeper".
  14877. + */
  14878. + if (!(wake_flags & WF_LOCK_SLEEPER))
  14879. + p->saved_state = TASK_RUNNING;
  14880. trace_sched_waking(p);
  14881. @@ -1982,53 +2138,6 @@
  14882. }
  14883. /**
  14884. - * try_to_wake_up_local - try to wake up a local task with rq lock held
  14885. - * @p: the thread to be awakened
  14886. - *
  14887. - * Put @p on the run-queue if it's not already there. The caller must
  14888. - * ensure that this_rq() is locked, @p is bound to this_rq() and not
  14889. - * the current task.
  14890. - */
  14891. -static void try_to_wake_up_local(struct task_struct *p)
  14892. -{
  14893. - struct rq *rq = task_rq(p);
  14894. -
  14895. - if (WARN_ON_ONCE(rq != this_rq()) ||
  14896. - WARN_ON_ONCE(p == current))
  14897. - return;
  14898. -
  14899. - lockdep_assert_held(&rq->lock);
  14900. -
  14901. - if (!raw_spin_trylock(&p->pi_lock)) {
  14902. - /*
  14903. - * This is OK, because current is on_cpu, which avoids it being
  14904. - * picked for load-balance and preemption/IRQs are still
  14905. - * disabled avoiding further scheduler activity on it and we've
  14906. - * not yet picked a replacement task.
  14907. - */
  14908. - lockdep_unpin_lock(&rq->lock);
  14909. - raw_spin_unlock(&rq->lock);
  14910. - raw_spin_lock(&p->pi_lock);
  14911. - raw_spin_lock(&rq->lock);
  14912. - lockdep_pin_lock(&rq->lock);
  14913. - }
  14914. -
  14915. - if (!(p->state & TASK_NORMAL))
  14916. - goto out;
  14917. -
  14918. - trace_sched_waking(p);
  14919. -
  14920. - if (!task_on_rq_queued(p))
  14921. - ttwu_activate(rq, p, ENQUEUE_WAKEUP);
  14922. -
  14923. - ttwu_do_wakeup(rq, p, 0);
  14924. - if (schedstat_enabled())
  14925. - ttwu_stat(p, smp_processor_id(), 0);
  14926. -out:
  14927. - raw_spin_unlock(&p->pi_lock);
  14928. -}
  14929. -
  14930. -/**
  14931. * wake_up_process - Wake up a specific process
  14932. * @p: The process to be woken up.
  14933. *
  14934. @@ -2046,6 +2155,18 @@
  14935. }
  14936. EXPORT_SYMBOL(wake_up_process);
  14937. +/**
  14938. + * wake_up_lock_sleeper - Wake up a specific process blocked on a "sleeping lock"
  14939. + * @p: The process to be woken up.
  14940. + *
  14941. + * Same as wake_up_process() above, but wake_flags=WF_LOCK_SLEEPER to indicate
  14942. + * the nature of the wakeup.
  14943. + */
  14944. +int wake_up_lock_sleeper(struct task_struct *p)
  14945. +{
  14946. + return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER);
  14947. +}
  14948. +
  14949. int wake_up_state(struct task_struct *p, unsigned int state)
  14950. {
  14951. return try_to_wake_up(p, state, 0);
  14952. @@ -2303,6 +2424,9 @@
  14953. p->on_cpu = 0;
  14954. #endif
  14955. init_task_preempt_count(p);
  14956. +#ifdef CONFIG_HAVE_PREEMPT_LAZY
  14957. + task_thread_info(p)->preempt_lazy_count = 0;
  14958. +#endif
  14959. #ifdef CONFIG_SMP
  14960. plist_node_init(&p->pushable_tasks, MAX_PRIO);
  14961. RB_CLEAR_NODE(&p->pushable_dl_tasks);
  14962. @@ -2627,8 +2751,12 @@
  14963. finish_arch_post_lock_switch();
  14964. fire_sched_in_preempt_notifiers(current);
  14965. + /*
  14966. + * We use mmdrop_delayed() here so we don't have to do the
  14967. + * full __mmdrop() when we are the last user.
  14968. + */
  14969. if (mm)
  14970. - mmdrop(mm);
  14971. + mmdrop_delayed(mm);
  14972. if (unlikely(prev_state == TASK_DEAD)) {
  14973. if (prev->sched_class->task_dead)
  14974. prev->sched_class->task_dead(prev);
  14975. @@ -3062,6 +3190,77 @@
  14976. schedstat_inc(this_rq(), sched_count);
  14977. }
  14978. +#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP)
  14979. +
  14980. +void migrate_disable(void)
  14981. +{
  14982. + struct task_struct *p = current;
  14983. +
  14984. + if (in_atomic() || irqs_disabled()) {
  14985. +#ifdef CONFIG_SCHED_DEBUG
  14986. + p->migrate_disable_atomic++;
  14987. +#endif
  14988. + return;
  14989. + }
  14990. +
  14991. +#ifdef CONFIG_SCHED_DEBUG
  14992. + if (unlikely(p->migrate_disable_atomic)) {
  14993. + tracing_off();
  14994. + WARN_ON_ONCE(1);
  14995. + }
  14996. +#endif
  14997. +
  14998. + if (p->migrate_disable) {
  14999. + p->migrate_disable++;
  15000. + return;
  15001. + }
  15002. +
  15003. + preempt_disable();
  15004. + preempt_lazy_disable();
  15005. + pin_current_cpu();
  15006. + p->migrate_disable = 1;
  15007. + preempt_enable();
  15008. +}
  15009. +EXPORT_SYMBOL(migrate_disable);
  15010. +
  15011. +void migrate_enable(void)
  15012. +{
  15013. + struct task_struct *p = current;
  15014. +
  15015. + if (in_atomic() || irqs_disabled()) {
  15016. +#ifdef CONFIG_SCHED_DEBUG
  15017. + p->migrate_disable_atomic--;
  15018. +#endif
  15019. + return;
  15020. + }
  15021. +
  15022. +#ifdef CONFIG_SCHED_DEBUG
  15023. + if (unlikely(p->migrate_disable_atomic)) {
  15024. + tracing_off();
  15025. + WARN_ON_ONCE(1);
  15026. + }
  15027. +#endif
  15028. + WARN_ON_ONCE(p->migrate_disable <= 0);
  15029. +
  15030. + if (p->migrate_disable > 1) {
  15031. + p->migrate_disable--;
  15032. + return;
  15033. + }
  15034. +
  15035. + preempt_disable();
  15036. + /*
  15037. + * Clearing migrate_disable causes tsk_cpus_allowed to
  15038. + * show the tasks original cpu affinity.
  15039. + */
  15040. + p->migrate_disable = 0;
  15041. +
  15042. + unpin_current_cpu();
  15043. + preempt_enable();
  15044. + preempt_lazy_enable();
  15045. +}
  15046. +EXPORT_SYMBOL(migrate_enable);
  15047. +#endif
  15048. +
  15049. /*
  15050. * Pick up the highest-prio task:
  15051. */
  15052. @@ -3188,19 +3387,6 @@
  15053. } else {
  15054. deactivate_task(rq, prev, DEQUEUE_SLEEP);
  15055. prev->on_rq = 0;
  15056. -
  15057. - /*
  15058. - * If a worker went to sleep, notify and ask workqueue
  15059. - * whether it wants to wake up a task to maintain
  15060. - * concurrency.
  15061. - */
  15062. - if (prev->flags & PF_WQ_WORKER) {
  15063. - struct task_struct *to_wakeup;
  15064. -
  15065. - to_wakeup = wq_worker_sleeping(prev);
  15066. - if (to_wakeup)
  15067. - try_to_wake_up_local(to_wakeup);
  15068. - }
  15069. }
  15070. switch_count = &prev->nvcsw;
  15071. }
  15072. @@ -3210,6 +3396,7 @@
  15073. next = pick_next_task(rq, prev);
  15074. clear_tsk_need_resched(prev);
  15075. + clear_tsk_need_resched_lazy(prev);
  15076. clear_preempt_need_resched();
  15077. rq->clock_skip_update = 0;
  15078. @@ -3231,9 +3418,20 @@
  15079. static inline void sched_submit_work(struct task_struct *tsk)
  15080. {
  15081. - if (!tsk->state || tsk_is_pi_blocked(tsk))
  15082. + if (!tsk->state)
  15083. return;
  15084. /*
  15085. + * If a worker went to sleep, notify and ask workqueue whether
  15086. + * it wants to wake up a task to maintain concurrency.
  15087. + */
  15088. + if (tsk->flags & PF_WQ_WORKER)
  15089. + wq_worker_sleeping(tsk);
  15090. +
  15091. +
  15092. + if (tsk_is_pi_blocked(tsk))
  15093. + return;
  15094. +
  15095. + /*
  15096. * If we are going to sleep and we have plugged IO queued,
  15097. * make sure to submit it to avoid deadlocks.
  15098. */
  15099. @@ -3241,6 +3439,12 @@
  15100. blk_schedule_flush_plug(tsk);
  15101. }
  15102. +static void sched_update_worker(struct task_struct *tsk)
  15103. +{
  15104. + if (tsk->flags & PF_WQ_WORKER)
  15105. + wq_worker_running(tsk);
  15106. +}
  15107. +
  15108. asmlinkage __visible void __sched schedule(void)
  15109. {
  15110. struct task_struct *tsk = current;
  15111. @@ -3251,6 +3455,7 @@
  15112. __schedule(false);
  15113. sched_preempt_enable_no_resched();
  15114. } while (need_resched());
  15115. + sched_update_worker(tsk);
  15116. }
  15117. EXPORT_SYMBOL(schedule);
  15118. @@ -3299,6 +3504,30 @@
  15119. } while (need_resched());
  15120. }
  15121. +#ifdef CONFIG_PREEMPT_LAZY
  15122. +/*
  15123. + * If TIF_NEED_RESCHED is then we allow to be scheduled away since this is
  15124. + * set by a RT task. Oterwise we try to avoid beeing scheduled out as long as
  15125. + * preempt_lazy_count counter >0.
  15126. + */
  15127. +static __always_inline int preemptible_lazy(void)
  15128. +{
  15129. + if (test_thread_flag(TIF_NEED_RESCHED))
  15130. + return 1;
  15131. + if (current_thread_info()->preempt_lazy_count)
  15132. + return 0;
  15133. + return 1;
  15134. +}
  15135. +
  15136. +#else
  15137. +
  15138. +static inline int preemptible_lazy(void)
  15139. +{
  15140. + return 1;
  15141. +}
  15142. +
  15143. +#endif
  15144. +
  15145. #ifdef CONFIG_PREEMPT
  15146. /*
  15147. * this is the entry point to schedule() from in-kernel preemption
  15148. @@ -3313,6 +3542,8 @@
  15149. */
  15150. if (likely(!preemptible()))
  15151. return;
  15152. + if (!preemptible_lazy())
  15153. + return;
  15154. preempt_schedule_common();
  15155. }
  15156. @@ -3339,6 +3570,8 @@
  15157. if (likely(!preemptible()))
  15158. return;
  15159. + if (!preemptible_lazy())
  15160. + return;
  15161. do {
  15162. preempt_disable_notrace();
  15163. @@ -3348,7 +3581,16 @@
  15164. * an infinite recursion.
  15165. */
  15166. prev_ctx = exception_enter();
  15167. + /*
  15168. + * The add/subtract must not be traced by the function
  15169. + * tracer. But we still want to account for the
  15170. + * preempt off latency tracer. Since the _notrace versions
  15171. + * of add/subtract skip the accounting for latency tracer
  15172. + * we must force it manually.
  15173. + */
  15174. + start_critical_timings();
  15175. __schedule(true);
  15176. + stop_critical_timings();
  15177. exception_exit(prev_ctx);
  15178. preempt_enable_no_resched_notrace();
  15179. @@ -4693,6 +4935,7 @@
  15180. }
  15181. EXPORT_SYMBOL(__cond_resched_lock);
  15182. +#ifndef CONFIG_PREEMPT_RT_FULL
  15183. int __sched __cond_resched_softirq(void)
  15184. {
  15185. BUG_ON(!in_softirq());
  15186. @@ -4706,6 +4949,7 @@
  15187. return 0;
  15188. }
  15189. EXPORT_SYMBOL(__cond_resched_softirq);
  15190. +#endif
  15191. /**
  15192. * yield - yield the current processor to other threads.
  15193. @@ -5074,7 +5318,9 @@
  15194. /* Set the preempt count _outside_ the spinlocks! */
  15195. init_idle_preempt_count(idle, cpu);
  15196. -
  15197. +#ifdef CONFIG_HAVE_PREEMPT_LAZY
  15198. + task_thread_info(idle)->preempt_lazy_count = 0;
  15199. +#endif
  15200. /*
  15201. * The idle tasks have their own, simple scheduling class:
  15202. */
  15203. @@ -5215,6 +5461,8 @@
  15204. #endif /* CONFIG_NUMA_BALANCING */
  15205. #ifdef CONFIG_HOTPLUG_CPU
  15206. +static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm);
  15207. +
  15208. /*
  15209. * Ensures that the idle task is using init_mm right before its cpu goes
  15210. * offline.
  15211. @@ -5229,7 +5477,11 @@
  15212. switch_mm(mm, &init_mm, current);
  15213. finish_arch_post_lock_switch();
  15214. }
  15215. - mmdrop(mm);
  15216. + /*
  15217. + * Defer the cleanup to an alive cpu. On RT we can neither
  15218. + * call mmdrop() nor mmdrop_delayed() from here.
  15219. + */
  15220. + per_cpu(idle_last_mm, smp_processor_id()) = mm;
  15221. }
  15222. /*
  15223. @@ -5425,6 +5677,10 @@
  15224. case CPU_DEAD:
  15225. calc_load_migrate(rq);
  15226. + if (per_cpu(idle_last_mm, cpu)) {
  15227. + mmdrop(per_cpu(idle_last_mm, cpu));
  15228. + per_cpu(idle_last_mm, cpu) = NULL;
  15229. + }
  15230. break;
  15231. #endif
  15232. }
  15233. @@ -7407,7 +7663,7 @@
  15234. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  15235. static inline int preempt_count_equals(int preempt_offset)
  15236. {
  15237. - int nested = preempt_count() + rcu_preempt_depth();
  15238. + int nested = preempt_count() + sched_rcu_preempt_depth();
  15239. return (nested == preempt_offset);
  15240. }
  15241. diff -Nur linux-4.6.6.orig/kernel/sched/cpudeadline.c linux-4.6.6/kernel/sched/cpudeadline.c
  15242. --- linux-4.6.6.orig/kernel/sched/cpudeadline.c 2016-08-10 12:54:57.000000000 +0200
  15243. +++ linux-4.6.6/kernel/sched/cpudeadline.c 2016-08-22 21:27:47.413893109 +0200
  15244. @@ -103,10 +103,10 @@
  15245. const struct sched_dl_entity *dl_se = &p->dl;
  15246. if (later_mask &&
  15247. - cpumask_and(later_mask, cp->free_cpus, &p->cpus_allowed)) {
  15248. + cpumask_and(later_mask, cp->free_cpus, tsk_cpus_allowed(p))) {
  15249. best_cpu = cpumask_any(later_mask);
  15250. goto out;
  15251. - } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) &&
  15252. + } else if (cpumask_test_cpu(cpudl_maximum(cp), tsk_cpus_allowed(p)) &&
  15253. dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
  15254. best_cpu = cpudl_maximum(cp);
  15255. if (later_mask)
  15256. diff -Nur linux-4.6.6.orig/kernel/sched/cpupri.c linux-4.6.6/kernel/sched/cpupri.c
  15257. --- linux-4.6.6.orig/kernel/sched/cpupri.c 2016-08-10 12:54:57.000000000 +0200
  15258. +++ linux-4.6.6/kernel/sched/cpupri.c 2016-08-22 21:27:47.413893109 +0200
  15259. @@ -103,11 +103,11 @@
  15260. if (skip)
  15261. continue;
  15262. - if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
  15263. + if (cpumask_any_and(tsk_cpus_allowed(p), vec->mask) >= nr_cpu_ids)
  15264. continue;
  15265. if (lowest_mask) {
  15266. - cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
  15267. + cpumask_and(lowest_mask, tsk_cpus_allowed(p), vec->mask);
  15268. /*
  15269. * We have to ensure that we have at least one bit
  15270. diff -Nur linux-4.6.6.orig/kernel/sched/deadline.c linux-4.6.6/kernel/sched/deadline.c
  15271. --- linux-4.6.6.orig/kernel/sched/deadline.c 2016-08-10 12:54:57.000000000 +0200
  15272. +++ linux-4.6.6/kernel/sched/deadline.c 2016-08-22 21:27:47.413893109 +0200
  15273. @@ -134,7 +134,7 @@
  15274. {
  15275. struct task_struct *p = dl_task_of(dl_se);
  15276. - if (p->nr_cpus_allowed > 1)
  15277. + if (tsk_nr_cpus_allowed(p) > 1)
  15278. dl_rq->dl_nr_migratory++;
  15279. update_dl_migration(dl_rq);
  15280. @@ -144,7 +144,7 @@
  15281. {
  15282. struct task_struct *p = dl_task_of(dl_se);
  15283. - if (p->nr_cpus_allowed > 1)
  15284. + if (tsk_nr_cpus_allowed(p) > 1)
  15285. dl_rq->dl_nr_migratory--;
  15286. update_dl_migration(dl_rq);
  15287. @@ -694,6 +694,7 @@
  15288. hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  15289. timer->function = dl_task_timer;
  15290. + timer->irqsafe = 1;
  15291. }
  15292. static
  15293. @@ -966,7 +967,7 @@
  15294. enqueue_dl_entity(&p->dl, pi_se, flags);
  15295. - if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
  15296. + if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
  15297. enqueue_pushable_dl_task(rq, p);
  15298. }
  15299. @@ -1040,9 +1041,9 @@
  15300. * try to make it stay here, it might be important.
  15301. */
  15302. if (unlikely(dl_task(curr)) &&
  15303. - (curr->nr_cpus_allowed < 2 ||
  15304. + (tsk_nr_cpus_allowed(curr) < 2 ||
  15305. !dl_entity_preempt(&p->dl, &curr->dl)) &&
  15306. - (p->nr_cpus_allowed > 1)) {
  15307. + (tsk_nr_cpus_allowed(p) > 1)) {
  15308. int target = find_later_rq(p);
  15309. if (target != -1 &&
  15310. @@ -1063,7 +1064,7 @@
  15311. * Current can't be migrated, useless to reschedule,
  15312. * let's hope p can move out.
  15313. */
  15314. - if (rq->curr->nr_cpus_allowed == 1 ||
  15315. + if (tsk_nr_cpus_allowed(rq->curr) == 1 ||
  15316. cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1)
  15317. return;
  15318. @@ -1071,7 +1072,7 @@
  15319. * p is migratable, so let's not schedule it and
  15320. * see if it is pushed or pulled somewhere else.
  15321. */
  15322. - if (p->nr_cpus_allowed != 1 &&
  15323. + if (tsk_nr_cpus_allowed(p) != 1 &&
  15324. cpudl_find(&rq->rd->cpudl, p, NULL) != -1)
  15325. return;
  15326. @@ -1185,7 +1186,7 @@
  15327. {
  15328. update_curr_dl(rq);
  15329. - if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)
  15330. + if (on_dl_rq(&p->dl) && tsk_nr_cpus_allowed(p) > 1)
  15331. enqueue_pushable_dl_task(rq, p);
  15332. }
  15333. @@ -1286,7 +1287,7 @@
  15334. if (unlikely(!later_mask))
  15335. return -1;
  15336. - if (task->nr_cpus_allowed == 1)
  15337. + if (tsk_nr_cpus_allowed(task) == 1)
  15338. return -1;
  15339. /*
  15340. @@ -1392,7 +1393,7 @@
  15341. if (double_lock_balance(rq, later_rq)) {
  15342. if (unlikely(task_rq(task) != rq ||
  15343. !cpumask_test_cpu(later_rq->cpu,
  15344. - &task->cpus_allowed) ||
  15345. + tsk_cpus_allowed(task)) ||
  15346. task_running(rq, task) ||
  15347. !dl_task(task) ||
  15348. !task_on_rq_queued(task))) {
  15349. @@ -1432,7 +1433,7 @@
  15350. BUG_ON(rq->cpu != task_cpu(p));
  15351. BUG_ON(task_current(rq, p));
  15352. - BUG_ON(p->nr_cpus_allowed <= 1);
  15353. + BUG_ON(tsk_nr_cpus_allowed(p) <= 1);
  15354. BUG_ON(!task_on_rq_queued(p));
  15355. BUG_ON(!dl_task(p));
  15356. @@ -1471,7 +1472,7 @@
  15357. */
  15358. if (dl_task(rq->curr) &&
  15359. dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) &&
  15360. - rq->curr->nr_cpus_allowed > 1) {
  15361. + tsk_nr_cpus_allowed(rq->curr) > 1) {
  15362. resched_curr(rq);
  15363. return 0;
  15364. }
  15365. @@ -1618,9 +1619,9 @@
  15366. {
  15367. if (!task_running(rq, p) &&
  15368. !test_tsk_need_resched(rq->curr) &&
  15369. - p->nr_cpus_allowed > 1 &&
  15370. + tsk_nr_cpus_allowed(p) > 1 &&
  15371. dl_task(rq->curr) &&
  15372. - (rq->curr->nr_cpus_allowed < 2 ||
  15373. + (tsk_nr_cpus_allowed(rq->curr) < 2 ||
  15374. !dl_entity_preempt(&p->dl, &rq->curr->dl))) {
  15375. push_dl_tasks(rq);
  15376. }
  15377. @@ -1724,7 +1725,7 @@
  15378. if (task_on_rq_queued(p) && rq->curr != p) {
  15379. #ifdef CONFIG_SMP
  15380. - if (p->nr_cpus_allowed > 1 && rq->dl.overloaded)
  15381. + if (tsk_nr_cpus_allowed(p) > 1 && rq->dl.overloaded)
  15382. queue_push_tasks(rq);
  15383. #else
  15384. if (dl_task(rq->curr))
  15385. diff -Nur linux-4.6.6.orig/kernel/sched/debug.c linux-4.6.6/kernel/sched/debug.c
  15386. --- linux-4.6.6.orig/kernel/sched/debug.c 2016-08-10 12:54:57.000000000 +0200
  15387. +++ linux-4.6.6/kernel/sched/debug.c 2016-08-22 21:27:47.413893109 +0200
  15388. @@ -559,6 +559,9 @@
  15389. P(rt_throttled);
  15390. PN(rt_time);
  15391. PN(rt_runtime);
  15392. +#ifdef CONFIG_SMP
  15393. + P(rt_nr_migratory);
  15394. +#endif
  15395. #undef PN
  15396. #undef P
  15397. @@ -954,6 +957,10 @@
  15398. #endif
  15399. P(policy);
  15400. P(prio);
  15401. +#ifdef CONFIG_PREEMPT_RT_FULL
  15402. + P(migrate_disable);
  15403. +#endif
  15404. + P(nr_cpus_allowed);
  15405. #undef PN
  15406. #undef __PN
  15407. #undef P
  15408. diff -Nur linux-4.6.6.orig/kernel/sched/fair.c linux-4.6.6/kernel/sched/fair.c
  15409. --- linux-4.6.6.orig/kernel/sched/fair.c 2016-08-10 12:54:57.000000000 +0200
  15410. +++ linux-4.6.6/kernel/sched/fair.c 2016-08-22 21:27:47.413893109 +0200
  15411. @@ -3333,7 +3333,7 @@
  15412. ideal_runtime = sched_slice(cfs_rq, curr);
  15413. delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
  15414. if (delta_exec > ideal_runtime) {
  15415. - resched_curr(rq_of(cfs_rq));
  15416. + resched_curr_lazy(rq_of(cfs_rq));
  15417. /*
  15418. * The current task ran long enough, ensure it doesn't get
  15419. * re-elected due to buddy favours.
  15420. @@ -3357,7 +3357,7 @@
  15421. return;
  15422. if (delta > ideal_runtime)
  15423. - resched_curr(rq_of(cfs_rq));
  15424. + resched_curr_lazy(rq_of(cfs_rq));
  15425. }
  15426. static void
  15427. @@ -3502,7 +3502,7 @@
  15428. * validating it and just reschedule.
  15429. */
  15430. if (queued) {
  15431. - resched_curr(rq_of(cfs_rq));
  15432. + resched_curr_lazy(rq_of(cfs_rq));
  15433. return;
  15434. }
  15435. /*
  15436. @@ -3684,7 +3684,7 @@
  15437. * hierarchy can be throttled
  15438. */
  15439. if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
  15440. - resched_curr(rq_of(cfs_rq));
  15441. + resched_curr_lazy(rq_of(cfs_rq));
  15442. }
  15443. static __always_inline
  15444. @@ -4296,7 +4296,7 @@
  15445. if (delta < 0) {
  15446. if (rq->curr == p)
  15447. - resched_curr(rq);
  15448. + resched_curr_lazy(rq);
  15449. return;
  15450. }
  15451. hrtick_start(rq, delta);
  15452. @@ -5441,7 +5441,7 @@
  15453. return;
  15454. preempt:
  15455. - resched_curr(rq);
  15456. + resched_curr_lazy(rq);
  15457. /*
  15458. * Only set the backward buddy when the current task is still
  15459. * on the rq. This can happen when a wakeup gets interleaved
  15460. @@ -8192,7 +8192,7 @@
  15461. * 'current' within the tree based on its new key value.
  15462. */
  15463. swap(curr->vruntime, se->vruntime);
  15464. - resched_curr(rq);
  15465. + resched_curr_lazy(rq);
  15466. }
  15467. se->vruntime -= cfs_rq->min_vruntime;
  15468. @@ -8217,7 +8217,7 @@
  15469. */
  15470. if (rq->curr == p) {
  15471. if (p->prio > oldprio)
  15472. - resched_curr(rq);
  15473. + resched_curr_lazy(rq);
  15474. } else
  15475. check_preempt_curr(rq, p, 0);
  15476. }
  15477. diff -Nur linux-4.6.6.orig/kernel/sched/features.h linux-4.6.6/kernel/sched/features.h
  15478. --- linux-4.6.6.orig/kernel/sched/features.h 2016-08-10 12:54:57.000000000 +0200
  15479. +++ linux-4.6.6/kernel/sched/features.h 2016-08-22 21:27:47.413893109 +0200
  15480. @@ -45,11 +45,19 @@
  15481. */
  15482. SCHED_FEAT(NONTASK_CAPACITY, true)
  15483. +#ifdef CONFIG_PREEMPT_RT_FULL
  15484. +SCHED_FEAT(TTWU_QUEUE, false)
  15485. +# ifdef CONFIG_PREEMPT_LAZY
  15486. +SCHED_FEAT(PREEMPT_LAZY, true)
  15487. +# endif
  15488. +#else
  15489. +
  15490. /*
  15491. * Queue remote wakeups on the target CPU and process them
  15492. * using the scheduler IPI. Reduces rq->lock contention/bounces.
  15493. */
  15494. SCHED_FEAT(TTWU_QUEUE, true)
  15495. +#endif
  15496. #ifdef HAVE_RT_PUSH_IPI
  15497. /*
  15498. diff -Nur linux-4.6.6.orig/kernel/sched/Makefile linux-4.6.6/kernel/sched/Makefile
  15499. --- linux-4.6.6.orig/kernel/sched/Makefile 2016-08-10 12:54:57.000000000 +0200
  15500. +++ linux-4.6.6/kernel/sched/Makefile 2016-08-22 21:27:47.405892600 +0200
  15501. @@ -17,7 +17,7 @@
  15502. obj-y += core.o loadavg.o clock.o cputime.o
  15503. obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
  15504. -obj-y += wait.o swait.o completion.o idle.o
  15505. +obj-y += wait.o swait.o swork.o completion.o idle.o
  15506. obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
  15507. obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
  15508. obj-$(CONFIG_SCHEDSTATS) += stats.o
  15509. diff -Nur linux-4.6.6.orig/kernel/sched/rt.c linux-4.6.6/kernel/sched/rt.c
  15510. --- linux-4.6.6.orig/kernel/sched/rt.c 2016-08-10 12:54:57.000000000 +0200
  15511. +++ linux-4.6.6/kernel/sched/rt.c 2016-08-22 21:27:47.417893368 +0200
  15512. @@ -47,6 +47,7 @@
  15513. hrtimer_init(&rt_b->rt_period_timer,
  15514. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  15515. + rt_b->rt_period_timer.irqsafe = 1;
  15516. rt_b->rt_period_timer.function = sched_rt_period_timer;
  15517. }
  15518. @@ -101,6 +102,7 @@
  15519. rt_rq->push_cpu = nr_cpu_ids;
  15520. raw_spin_lock_init(&rt_rq->push_lock);
  15521. init_irq_work(&rt_rq->push_work, push_irq_work_func);
  15522. + rt_rq->push_work.flags |= IRQ_WORK_HARD_IRQ;
  15523. #endif
  15524. #endif /* CONFIG_SMP */
  15525. /* We start is dequeued state, because no RT tasks are queued */
  15526. @@ -334,7 +336,7 @@
  15527. rt_rq = &rq_of_rt_rq(rt_rq)->rt;
  15528. rt_rq->rt_nr_total++;
  15529. - if (p->nr_cpus_allowed > 1)
  15530. + if (tsk_nr_cpus_allowed(p) > 1)
  15531. rt_rq->rt_nr_migratory++;
  15532. update_rt_migration(rt_rq);
  15533. @@ -351,7 +353,7 @@
  15534. rt_rq = &rq_of_rt_rq(rt_rq)->rt;
  15535. rt_rq->rt_nr_total--;
  15536. - if (p->nr_cpus_allowed > 1)
  15537. + if (tsk_nr_cpus_allowed(p) > 1)
  15538. rt_rq->rt_nr_migratory--;
  15539. update_rt_migration(rt_rq);
  15540. @@ -1324,7 +1326,7 @@
  15541. enqueue_rt_entity(rt_se, flags);
  15542. - if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
  15543. + if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
  15544. enqueue_pushable_task(rq, p);
  15545. }
  15546. @@ -1413,7 +1415,7 @@
  15547. * will have to sort it out.
  15548. */
  15549. if (curr && unlikely(rt_task(curr)) &&
  15550. - (curr->nr_cpus_allowed < 2 ||
  15551. + (tsk_nr_cpus_allowed(curr) < 2 ||
  15552. curr->prio <= p->prio)) {
  15553. int target = find_lowest_rq(p);
  15554. @@ -1437,7 +1439,7 @@
  15555. * Current can't be migrated, useless to reschedule,
  15556. * let's hope p can move out.
  15557. */
  15558. - if (rq->curr->nr_cpus_allowed == 1 ||
  15559. + if (tsk_nr_cpus_allowed(rq->curr) == 1 ||
  15560. !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
  15561. return;
  15562. @@ -1445,7 +1447,7 @@
  15563. * p is migratable, so let's not schedule it and
  15564. * see if it is pushed or pulled somewhere else.
  15565. */
  15566. - if (p->nr_cpus_allowed != 1
  15567. + if (tsk_nr_cpus_allowed(p) != 1
  15568. && cpupri_find(&rq->rd->cpupri, p, NULL))
  15569. return;
  15570. @@ -1579,7 +1581,7 @@
  15571. * The previous task needs to be made eligible for pushing
  15572. * if it is still active
  15573. */
  15574. - if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
  15575. + if (on_rt_rq(&p->rt) && tsk_nr_cpus_allowed(p) > 1)
  15576. enqueue_pushable_task(rq, p);
  15577. }
  15578. @@ -1629,7 +1631,7 @@
  15579. if (unlikely(!lowest_mask))
  15580. return -1;
  15581. - if (task->nr_cpus_allowed == 1)
  15582. + if (tsk_nr_cpus_allowed(task) == 1)
  15583. return -1; /* No other targets possible */
  15584. if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
  15585. @@ -1762,7 +1764,7 @@
  15586. BUG_ON(rq->cpu != task_cpu(p));
  15587. BUG_ON(task_current(rq, p));
  15588. - BUG_ON(p->nr_cpus_allowed <= 1);
  15589. + BUG_ON(tsk_nr_cpus_allowed(p) <= 1);
  15590. BUG_ON(!task_on_rq_queued(p));
  15591. BUG_ON(!rt_task(p));
  15592. @@ -2122,9 +2124,9 @@
  15593. {
  15594. if (!task_running(rq, p) &&
  15595. !test_tsk_need_resched(rq->curr) &&
  15596. - p->nr_cpus_allowed > 1 &&
  15597. + tsk_nr_cpus_allowed(p) > 1 &&
  15598. (dl_task(rq->curr) || rt_task(rq->curr)) &&
  15599. - (rq->curr->nr_cpus_allowed < 2 ||
  15600. + (tsk_nr_cpus_allowed(rq->curr) < 2 ||
  15601. rq->curr->prio <= p->prio))
  15602. push_rt_tasks(rq);
  15603. }
  15604. @@ -2197,7 +2199,7 @@
  15605. */
  15606. if (task_on_rq_queued(p) && rq->curr != p) {
  15607. #ifdef CONFIG_SMP
  15608. - if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
  15609. + if (tsk_nr_cpus_allowed(p) > 1 && rq->rt.overloaded)
  15610. queue_push_tasks(rq);
  15611. #else
  15612. if (p->prio < rq->curr->prio)
  15613. diff -Nur linux-4.6.6.orig/kernel/sched/sched.h linux-4.6.6/kernel/sched/sched.h
  15614. --- linux-4.6.6.orig/kernel/sched/sched.h 2016-08-10 12:54:57.000000000 +0200
  15615. +++ linux-4.6.6/kernel/sched/sched.h 2016-08-22 21:27:47.417893368 +0200
  15616. @@ -1128,6 +1128,7 @@
  15617. #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */
  15618. #define WF_FORK 0x02 /* child wakeup after fork */
  15619. #define WF_MIGRATED 0x4 /* internal use, task got migrated */
  15620. +#define WF_LOCK_SLEEPER 0x08 /* wakeup spinlock "sleeper" */
  15621. /*
  15622. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  15623. @@ -1303,6 +1304,15 @@
  15624. extern void resched_curr(struct rq *rq);
  15625. extern void resched_cpu(int cpu);
  15626. +#ifdef CONFIG_PREEMPT_LAZY
  15627. +extern void resched_curr_lazy(struct rq *rq);
  15628. +#else
  15629. +static inline void resched_curr_lazy(struct rq *rq)
  15630. +{
  15631. + resched_curr(rq);
  15632. +}
  15633. +#endif
  15634. +
  15635. extern struct rt_bandwidth def_rt_bandwidth;
  15636. extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
  15637. diff -Nur linux-4.6.6.orig/kernel/sched/swait.c linux-4.6.6/kernel/sched/swait.c
  15638. --- linux-4.6.6.orig/kernel/sched/swait.c 2016-08-10 12:54:57.000000000 +0200
  15639. +++ linux-4.6.6/kernel/sched/swait.c 2016-08-22 21:27:47.417893368 +0200
  15640. @@ -1,5 +1,6 @@
  15641. #include <linux/sched.h>
  15642. #include <linux/swait.h>
  15643. +#include <linux/suspend.h>
  15644. void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
  15645. struct lock_class_key *key)
  15646. @@ -29,6 +30,25 @@
  15647. }
  15648. EXPORT_SYMBOL(swake_up_locked);
  15649. +void swake_up_all_locked(struct swait_queue_head *q)
  15650. +{
  15651. + struct swait_queue *curr;
  15652. + int wakes = 0;
  15653. +
  15654. + while (!list_empty(&q->task_list)) {
  15655. +
  15656. + curr = list_first_entry(&q->task_list, typeof(*curr),
  15657. + task_list);
  15658. + wake_up_process(curr->task);
  15659. + list_del_init(&curr->task_list);
  15660. + wakes++;
  15661. + }
  15662. + if (pm_in_action)
  15663. + return;
  15664. + WARN(wakes > 2, "complete_all() with %d waiters\n", wakes);
  15665. +}
  15666. +EXPORT_SYMBOL(swake_up_all_locked);
  15667. +
  15668. void swake_up(struct swait_queue_head *q)
  15669. {
  15670. unsigned long flags;
  15671. diff -Nur linux-4.6.6.orig/kernel/sched/swork.c linux-4.6.6/kernel/sched/swork.c
  15672. --- linux-4.6.6.orig/kernel/sched/swork.c 1970-01-01 01:00:00.000000000 +0100
  15673. +++ linux-4.6.6/kernel/sched/swork.c 2016-08-22 21:27:47.417893368 +0200
  15674. @@ -0,0 +1,173 @@
  15675. +/*
  15676. + * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de
  15677. + *
  15678. + * Provides a framework for enqueuing callbacks from irq context
  15679. + * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context.
  15680. + */
  15681. +
  15682. +#include <linux/swait.h>
  15683. +#include <linux/swork.h>
  15684. +#include <linux/kthread.h>
  15685. +#include <linux/slab.h>
  15686. +#include <linux/spinlock.h>
  15687. +#include <linux/export.h>
  15688. +
  15689. +#define SWORK_EVENT_PENDING (1 << 0)
  15690. +
  15691. +static DEFINE_MUTEX(worker_mutex);
  15692. +static struct sworker *glob_worker;
  15693. +
  15694. +struct sworker {
  15695. + struct list_head events;
  15696. + struct swait_queue_head wq;
  15697. +
  15698. + raw_spinlock_t lock;
  15699. +
  15700. + struct task_struct *task;
  15701. + int refs;
  15702. +};
  15703. +
  15704. +static bool swork_readable(struct sworker *worker)
  15705. +{
  15706. + bool r;
  15707. +
  15708. + if (kthread_should_stop())
  15709. + return true;
  15710. +
  15711. + raw_spin_lock_irq(&worker->lock);
  15712. + r = !list_empty(&worker->events);
  15713. + raw_spin_unlock_irq(&worker->lock);
  15714. +
  15715. + return r;
  15716. +}
  15717. +
  15718. +static int swork_kthread(void *arg)
  15719. +{
  15720. + struct sworker *worker = arg;
  15721. +
  15722. + for (;;) {
  15723. + swait_event_interruptible(worker->wq,
  15724. + swork_readable(worker));
  15725. + if (kthread_should_stop())
  15726. + break;
  15727. +
  15728. + raw_spin_lock_irq(&worker->lock);
  15729. + while (!list_empty(&worker->events)) {
  15730. + struct swork_event *sev;
  15731. +
  15732. + sev = list_first_entry(&worker->events,
  15733. + struct swork_event, item);
  15734. + list_del(&sev->item);
  15735. + raw_spin_unlock_irq(&worker->lock);
  15736. +
  15737. + WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING,
  15738. + &sev->flags));
  15739. + sev->func(sev);
  15740. + raw_spin_lock_irq(&worker->lock);
  15741. + }
  15742. + raw_spin_unlock_irq(&worker->lock);
  15743. + }
  15744. + return 0;
  15745. +}
  15746. +
  15747. +static struct sworker *swork_create(void)
  15748. +{
  15749. + struct sworker *worker;
  15750. +
  15751. + worker = kzalloc(sizeof(*worker), GFP_KERNEL);
  15752. + if (!worker)
  15753. + return ERR_PTR(-ENOMEM);
  15754. +
  15755. + INIT_LIST_HEAD(&worker->events);
  15756. + raw_spin_lock_init(&worker->lock);
  15757. + init_swait_queue_head(&worker->wq);
  15758. +
  15759. + worker->task = kthread_run(swork_kthread, worker, "kswork");
  15760. + if (IS_ERR(worker->task)) {
  15761. + kfree(worker);
  15762. + return ERR_PTR(-ENOMEM);
  15763. + }
  15764. +
  15765. + return worker;
  15766. +}
  15767. +
  15768. +static void swork_destroy(struct sworker *worker)
  15769. +{
  15770. + kthread_stop(worker->task);
  15771. +
  15772. + WARN_ON(!list_empty(&worker->events));
  15773. + kfree(worker);
  15774. +}
  15775. +
  15776. +/**
  15777. + * swork_queue - queue swork
  15778. + *
  15779. + * Returns %false if @work was already on a queue, %true otherwise.
  15780. + *
  15781. + * The work is queued and processed on a random CPU
  15782. + */
  15783. +bool swork_queue(struct swork_event *sev)
  15784. +{
  15785. + unsigned long flags;
  15786. +
  15787. + if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags))
  15788. + return false;
  15789. +
  15790. + raw_spin_lock_irqsave(&glob_worker->lock, flags);
  15791. + list_add_tail(&sev->item, &glob_worker->events);
  15792. + raw_spin_unlock_irqrestore(&glob_worker->lock, flags);
  15793. +
  15794. + swake_up(&glob_worker->wq);
  15795. + return true;
  15796. +}
  15797. +EXPORT_SYMBOL_GPL(swork_queue);
  15798. +
  15799. +/**
  15800. + * swork_get - get an instance of the sworker
  15801. + *
  15802. + * Returns an negative error code if the initialization if the worker did not
  15803. + * work, %0 otherwise.
  15804. + *
  15805. + */
  15806. +int swork_get(void)
  15807. +{
  15808. + struct sworker *worker;
  15809. +
  15810. + mutex_lock(&worker_mutex);
  15811. + if (!glob_worker) {
  15812. + worker = swork_create();
  15813. + if (IS_ERR(worker)) {
  15814. + mutex_unlock(&worker_mutex);
  15815. + return -ENOMEM;
  15816. + }
  15817. +
  15818. + glob_worker = worker;
  15819. + }
  15820. +
  15821. + glob_worker->refs++;
  15822. + mutex_unlock(&worker_mutex);
  15823. +
  15824. + return 0;
  15825. +}
  15826. +EXPORT_SYMBOL_GPL(swork_get);
  15827. +
  15828. +/**
  15829. + * swork_put - puts an instance of the sworker
  15830. + *
  15831. + * Will destroy the sworker thread. This function must not be called until all
  15832. + * queued events have been completed.
  15833. + */
  15834. +void swork_put(void)
  15835. +{
  15836. + mutex_lock(&worker_mutex);
  15837. +
  15838. + glob_worker->refs--;
  15839. + if (glob_worker->refs > 0)
  15840. + goto out;
  15841. +
  15842. + swork_destroy(glob_worker);
  15843. + glob_worker = NULL;
  15844. +out:
  15845. + mutex_unlock(&worker_mutex);
  15846. +}
  15847. +EXPORT_SYMBOL_GPL(swork_put);
  15848. diff -Nur linux-4.6.6.orig/kernel/signal.c linux-4.6.6/kernel/signal.c
  15849. --- linux-4.6.6.orig/kernel/signal.c 2016-08-10 12:54:57.000000000 +0200
  15850. +++ linux-4.6.6/kernel/signal.c 2016-08-22 21:27:47.417893368 +0200
  15851. @@ -14,6 +14,7 @@
  15852. #include <linux/export.h>
  15853. #include <linux/init.h>
  15854. #include <linux/sched.h>
  15855. +#include <linux/sched/rt.h>
  15856. #include <linux/fs.h>
  15857. #include <linux/tty.h>
  15858. #include <linux/binfmts.h>
  15859. @@ -352,13 +353,30 @@
  15860. return false;
  15861. }
  15862. +static inline struct sigqueue *get_task_cache(struct task_struct *t)
  15863. +{
  15864. + struct sigqueue *q = t->sigqueue_cache;
  15865. +
  15866. + if (cmpxchg(&t->sigqueue_cache, q, NULL) != q)
  15867. + return NULL;
  15868. + return q;
  15869. +}
  15870. +
  15871. +static inline int put_task_cache(struct task_struct *t, struct sigqueue *q)
  15872. +{
  15873. + if (cmpxchg(&t->sigqueue_cache, NULL, q) == NULL)
  15874. + return 0;
  15875. + return 1;
  15876. +}
  15877. +
  15878. /*
  15879. * allocate a new signal queue record
  15880. * - this may be called without locks if and only if t == current, otherwise an
  15881. * appropriate lock must be held to stop the target task from exiting
  15882. */
  15883. static struct sigqueue *
  15884. -__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit)
  15885. +__sigqueue_do_alloc(int sig, struct task_struct *t, gfp_t flags,
  15886. + int override_rlimit, int fromslab)
  15887. {
  15888. struct sigqueue *q = NULL;
  15889. struct user_struct *user;
  15890. @@ -375,7 +393,10 @@
  15891. if (override_rlimit ||
  15892. atomic_read(&user->sigpending) <=
  15893. task_rlimit(t, RLIMIT_SIGPENDING)) {
  15894. - q = kmem_cache_alloc(sigqueue_cachep, flags);
  15895. + if (!fromslab)
  15896. + q = get_task_cache(t);
  15897. + if (!q)
  15898. + q = kmem_cache_alloc(sigqueue_cachep, flags);
  15899. } else {
  15900. print_dropped_signal(sig);
  15901. }
  15902. @@ -392,6 +413,13 @@
  15903. return q;
  15904. }
  15905. +static struct sigqueue *
  15906. +__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags,
  15907. + int override_rlimit)
  15908. +{
  15909. + return __sigqueue_do_alloc(sig, t, flags, override_rlimit, 0);
  15910. +}
  15911. +
  15912. static void __sigqueue_free(struct sigqueue *q)
  15913. {
  15914. if (q->flags & SIGQUEUE_PREALLOC)
  15915. @@ -401,6 +429,21 @@
  15916. kmem_cache_free(sigqueue_cachep, q);
  15917. }
  15918. +static void sigqueue_free_current(struct sigqueue *q)
  15919. +{
  15920. + struct user_struct *up;
  15921. +
  15922. + if (q->flags & SIGQUEUE_PREALLOC)
  15923. + return;
  15924. +
  15925. + up = q->user;
  15926. + if (rt_prio(current->normal_prio) && !put_task_cache(current, q)) {
  15927. + atomic_dec(&up->sigpending);
  15928. + free_uid(up);
  15929. + } else
  15930. + __sigqueue_free(q);
  15931. +}
  15932. +
  15933. void flush_sigqueue(struct sigpending *queue)
  15934. {
  15935. struct sigqueue *q;
  15936. @@ -414,6 +457,21 @@
  15937. }
  15938. /*
  15939. + * Called from __exit_signal. Flush tsk->pending and
  15940. + * tsk->sigqueue_cache
  15941. + */
  15942. +void flush_task_sigqueue(struct task_struct *tsk)
  15943. +{
  15944. + struct sigqueue *q;
  15945. +
  15946. + flush_sigqueue(&tsk->pending);
  15947. +
  15948. + q = get_task_cache(tsk);
  15949. + if (q)
  15950. + kmem_cache_free(sigqueue_cachep, q);
  15951. +}
  15952. +
  15953. +/*
  15954. * Flush all pending signals for this kthread.
  15955. */
  15956. void flush_signals(struct task_struct *t)
  15957. @@ -525,7 +583,7 @@
  15958. still_pending:
  15959. list_del_init(&first->list);
  15960. copy_siginfo(info, &first->info);
  15961. - __sigqueue_free(first);
  15962. + sigqueue_free_current(first);
  15963. } else {
  15964. /*
  15965. * Ok, it wasn't in the queue. This must be
  15966. @@ -560,6 +618,8 @@
  15967. {
  15968. int signr;
  15969. + WARN_ON_ONCE(tsk != current);
  15970. +
  15971. /* We only dequeue private signals from ourselves, we don't let
  15972. * signalfd steal them
  15973. */
  15974. @@ -1156,8 +1216,8 @@
  15975. * We don't want to have recursive SIGSEGV's etc, for example,
  15976. * that is why we also clear SIGNAL_UNKILLABLE.
  15977. */
  15978. -int
  15979. -force_sig_info(int sig, struct siginfo *info, struct task_struct *t)
  15980. +static int
  15981. +do_force_sig_info(int sig, struct siginfo *info, struct task_struct *t)
  15982. {
  15983. unsigned long int flags;
  15984. int ret, blocked, ignored;
  15985. @@ -1182,6 +1242,39 @@
  15986. return ret;
  15987. }
  15988. +int force_sig_info(int sig, struct siginfo *info, struct task_struct *t)
  15989. +{
  15990. +/*
  15991. + * On some archs, PREEMPT_RT has to delay sending a signal from a trap
  15992. + * since it can not enable preemption, and the signal code's spin_locks
  15993. + * turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME which will
  15994. + * send the signal on exit of the trap.
  15995. + */
  15996. +#ifdef ARCH_RT_DELAYS_SIGNAL_SEND
  15997. + if (in_atomic()) {
  15998. + if (WARN_ON_ONCE(t != current))
  15999. + return 0;
  16000. + if (WARN_ON_ONCE(t->forced_info.si_signo))
  16001. + return 0;
  16002. +
  16003. + if (is_si_special(info)) {
  16004. + WARN_ON_ONCE(info != SEND_SIG_PRIV);
  16005. + t->forced_info.si_signo = sig;
  16006. + t->forced_info.si_errno = 0;
  16007. + t->forced_info.si_code = SI_KERNEL;
  16008. + t->forced_info.si_pid = 0;
  16009. + t->forced_info.si_uid = 0;
  16010. + } else {
  16011. + t->forced_info = *info;
  16012. + }
  16013. +
  16014. + set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
  16015. + return 0;
  16016. + }
  16017. +#endif
  16018. + return do_force_sig_info(sig, info, t);
  16019. +}
  16020. +
  16021. /*
  16022. * Nuke all other threads in the group.
  16023. */
  16024. @@ -1216,12 +1309,12 @@
  16025. * Disable interrupts early to avoid deadlocks.
  16026. * See rcu_read_unlock() comment header for details.
  16027. */
  16028. - local_irq_save(*flags);
  16029. + local_irq_save_nort(*flags);
  16030. rcu_read_lock();
  16031. sighand = rcu_dereference(tsk->sighand);
  16032. if (unlikely(sighand == NULL)) {
  16033. rcu_read_unlock();
  16034. - local_irq_restore(*flags);
  16035. + local_irq_restore_nort(*flags);
  16036. break;
  16037. }
  16038. /*
  16039. @@ -1242,7 +1335,7 @@
  16040. }
  16041. spin_unlock(&sighand->siglock);
  16042. rcu_read_unlock();
  16043. - local_irq_restore(*flags);
  16044. + local_irq_restore_nort(*flags);
  16045. }
  16046. return sighand;
  16047. @@ -1485,7 +1578,8 @@
  16048. */
  16049. struct sigqueue *sigqueue_alloc(void)
  16050. {
  16051. - struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0);
  16052. + /* Preallocated sigqueue objects always from the slabcache ! */
  16053. + struct sigqueue *q = __sigqueue_do_alloc(-1, current, GFP_KERNEL, 0, 1);
  16054. if (q)
  16055. q->flags |= SIGQUEUE_PREALLOC;
  16056. @@ -1846,15 +1940,7 @@
  16057. if (gstop_done && ptrace_reparented(current))
  16058. do_notify_parent_cldstop(current, false, why);
  16059. - /*
  16060. - * Don't want to allow preemption here, because
  16061. - * sys_ptrace() needs this task to be inactive.
  16062. - *
  16063. - * XXX: implement read_unlock_no_resched().
  16064. - */
  16065. - preempt_disable();
  16066. read_unlock(&tasklist_lock);
  16067. - preempt_enable_no_resched();
  16068. freezable_schedule();
  16069. } else {
  16070. /*
  16071. @@ -2751,23 +2837,18 @@
  16072. * @ts: upper bound on process time suspension
  16073. */
  16074. int do_sigtimedwait(const sigset_t *which, siginfo_t *info,
  16075. - const struct timespec *ts)
  16076. + const struct timespec *ts)
  16077. {
  16078. + ktime_t *to = NULL, timeout = { .tv64 = KTIME_MAX };
  16079. struct task_struct *tsk = current;
  16080. - long timeout = MAX_SCHEDULE_TIMEOUT;
  16081. sigset_t mask = *which;
  16082. - int sig;
  16083. + int sig, ret = 0;
  16084. if (ts) {
  16085. if (!timespec_valid(ts))
  16086. return -EINVAL;
  16087. - timeout = timespec_to_jiffies(ts);
  16088. - /*
  16089. - * We can be close to the next tick, add another one
  16090. - * to ensure we will wait at least the time asked for.
  16091. - */
  16092. - if (ts->tv_sec || ts->tv_nsec)
  16093. - timeout++;
  16094. + timeout = timespec_to_ktime(*ts);
  16095. + to = &timeout;
  16096. }
  16097. /*
  16098. @@ -2778,7 +2859,7 @@
  16099. spin_lock_irq(&tsk->sighand->siglock);
  16100. sig = dequeue_signal(tsk, &mask, info);
  16101. - if (!sig && timeout) {
  16102. + if (!sig && timeout.tv64) {
  16103. /*
  16104. * None ready, temporarily unblock those we're interested
  16105. * while we are sleeping in so that we'll be awakened when
  16106. @@ -2790,8 +2871,9 @@
  16107. recalc_sigpending();
  16108. spin_unlock_irq(&tsk->sighand->siglock);
  16109. - timeout = freezable_schedule_timeout_interruptible(timeout);
  16110. -
  16111. + __set_current_state(TASK_INTERRUPTIBLE);
  16112. + ret = freezable_schedule_hrtimeout_range(to, tsk->timer_slack_ns,
  16113. + HRTIMER_MODE_REL);
  16114. spin_lock_irq(&tsk->sighand->siglock);
  16115. __set_task_blocked(tsk, &tsk->real_blocked);
  16116. sigemptyset(&tsk->real_blocked);
  16117. @@ -2801,7 +2883,7 @@
  16118. if (sig)
  16119. return sig;
  16120. - return timeout ? -EINTR : -EAGAIN;
  16121. + return ret ? -EINTR : -EAGAIN;
  16122. }
  16123. /**
  16124. diff -Nur linux-4.6.6.orig/kernel/softirq.c linux-4.6.6/kernel/softirq.c
  16125. --- linux-4.6.6.orig/kernel/softirq.c 2016-08-10 12:54:57.000000000 +0200
  16126. +++ linux-4.6.6/kernel/softirq.c 2016-08-22 21:27:47.417893368 +0200
  16127. @@ -21,10 +21,12 @@
  16128. #include <linux/freezer.h>
  16129. #include <linux/kthread.h>
  16130. #include <linux/rcupdate.h>
  16131. +#include <linux/delay.h>
  16132. #include <linux/ftrace.h>
  16133. #include <linux/smp.h>
  16134. #include <linux/smpboot.h>
  16135. #include <linux/tick.h>
  16136. +#include <linux/locallock.h>
  16137. #include <linux/irq.h>
  16138. #define CREATE_TRACE_POINTS
  16139. @@ -56,12 +58,108 @@
  16140. static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  16141. DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  16142. +#ifdef CONFIG_PREEMPT_RT_FULL
  16143. +#define TIMER_SOFTIRQS ((1 << TIMER_SOFTIRQ) | (1 << HRTIMER_SOFTIRQ))
  16144. +DEFINE_PER_CPU(struct task_struct *, ktimer_softirqd);
  16145. +#endif
  16146. const char * const softirq_to_name[NR_SOFTIRQS] = {
  16147. "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  16148. "TASKLET", "SCHED", "HRTIMER", "RCU"
  16149. };
  16150. +#ifdef CONFIG_NO_HZ_COMMON
  16151. +# ifdef CONFIG_PREEMPT_RT_FULL
  16152. +
  16153. +struct softirq_runner {
  16154. + struct task_struct *runner[NR_SOFTIRQS];
  16155. +};
  16156. +
  16157. +static DEFINE_PER_CPU(struct softirq_runner, softirq_runners);
  16158. +
  16159. +static inline void softirq_set_runner(unsigned int sirq)
  16160. +{
  16161. + struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
  16162. +
  16163. + sr->runner[sirq] = current;
  16164. +}
  16165. +
  16166. +static inline void softirq_clr_runner(unsigned int sirq)
  16167. +{
  16168. + struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
  16169. +
  16170. + sr->runner[sirq] = NULL;
  16171. +}
  16172. +
  16173. +/*
  16174. + * On preempt-rt a softirq running context might be blocked on a
  16175. + * lock. There might be no other runnable task on this CPU because the
  16176. + * lock owner runs on some other CPU. So we have to go into idle with
  16177. + * the pending bit set. Therefor we need to check this otherwise we
  16178. + * warn about false positives which confuses users and defeats the
  16179. + * whole purpose of this test.
  16180. + *
  16181. + * This code is called with interrupts disabled.
  16182. + */
  16183. +void softirq_check_pending_idle(void)
  16184. +{
  16185. + static int rate_limit;
  16186. + struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
  16187. + u32 warnpending;
  16188. + int i;
  16189. +
  16190. + if (rate_limit >= 10)
  16191. + return;
  16192. +
  16193. + warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK;
  16194. + for (i = 0; i < NR_SOFTIRQS; i++) {
  16195. + struct task_struct *tsk = sr->runner[i];
  16196. +
  16197. + /*
  16198. + * The wakeup code in rtmutex.c wakes up the task
  16199. + * _before_ it sets pi_blocked_on to NULL under
  16200. + * tsk->pi_lock. So we need to check for both: state
  16201. + * and pi_blocked_on.
  16202. + */
  16203. + if (tsk) {
  16204. + raw_spin_lock(&tsk->pi_lock);
  16205. + if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) {
  16206. + /* Clear all bits pending in that task */
  16207. + warnpending &= ~(tsk->softirqs_raised);
  16208. + warnpending &= ~(1 << i);
  16209. + }
  16210. + raw_spin_unlock(&tsk->pi_lock);
  16211. + }
  16212. + }
  16213. +
  16214. + if (warnpending) {
  16215. + printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
  16216. + warnpending);
  16217. + rate_limit++;
  16218. + }
  16219. +}
  16220. +# else
  16221. +/*
  16222. + * On !PREEMPT_RT we just printk rate limited:
  16223. + */
  16224. +void softirq_check_pending_idle(void)
  16225. +{
  16226. + static int rate_limit;
  16227. +
  16228. + if (rate_limit < 10 &&
  16229. + (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
  16230. + printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
  16231. + local_softirq_pending());
  16232. + rate_limit++;
  16233. + }
  16234. +}
  16235. +# endif
  16236. +
  16237. +#else /* !CONFIG_NO_HZ_COMMON */
  16238. +static inline void softirq_set_runner(unsigned int sirq) { }
  16239. +static inline void softirq_clr_runner(unsigned int sirq) { }
  16240. +#endif
  16241. +
  16242. /*
  16243. * we cannot loop indefinitely here to avoid userspace starvation,
  16244. * but we also don't want to introduce a worst case 1/HZ latency
  16245. @@ -77,6 +175,79 @@
  16246. wake_up_process(tsk);
  16247. }
  16248. +#ifdef CONFIG_PREEMPT_RT_FULL
  16249. +static void wakeup_timer_softirqd(void)
  16250. +{
  16251. + /* Interrupts are disabled: no need to stop preemption */
  16252. + struct task_struct *tsk = __this_cpu_read(ktimer_softirqd);
  16253. +
  16254. + if (tsk && tsk->state != TASK_RUNNING)
  16255. + wake_up_process(tsk);
  16256. +}
  16257. +#endif
  16258. +
  16259. +static void handle_softirq(unsigned int vec_nr)
  16260. +{
  16261. + struct softirq_action *h = softirq_vec + vec_nr;
  16262. + int prev_count;
  16263. +
  16264. + prev_count = preempt_count();
  16265. +
  16266. + kstat_incr_softirqs_this_cpu(vec_nr);
  16267. +
  16268. + trace_softirq_entry(vec_nr);
  16269. + h->action(h);
  16270. + trace_softirq_exit(vec_nr);
  16271. + if (unlikely(prev_count != preempt_count())) {
  16272. + pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
  16273. + vec_nr, softirq_to_name[vec_nr], h->action,
  16274. + prev_count, preempt_count());
  16275. + preempt_count_set(prev_count);
  16276. + }
  16277. +}
  16278. +
  16279. +#ifndef CONFIG_PREEMPT_RT_FULL
  16280. +static inline int ksoftirqd_softirq_pending(void)
  16281. +{
  16282. + return local_softirq_pending();
  16283. +}
  16284. +
  16285. +static void handle_pending_softirqs(u32 pending)
  16286. +{
  16287. + struct softirq_action *h = softirq_vec;
  16288. + int softirq_bit;
  16289. +
  16290. + local_irq_enable();
  16291. +
  16292. + h = softirq_vec;
  16293. +
  16294. + while ((softirq_bit = ffs(pending))) {
  16295. + unsigned int vec_nr;
  16296. +
  16297. + h += softirq_bit - 1;
  16298. + vec_nr = h - softirq_vec;
  16299. + handle_softirq(vec_nr);
  16300. +
  16301. + h++;
  16302. + pending >>= softirq_bit;
  16303. + }
  16304. +
  16305. + rcu_bh_qs();
  16306. + local_irq_disable();
  16307. +}
  16308. +
  16309. +static void run_ksoftirqd(unsigned int cpu)
  16310. +{
  16311. + local_irq_disable();
  16312. + if (ksoftirqd_softirq_pending()) {
  16313. + __do_softirq();
  16314. + local_irq_enable();
  16315. + cond_resched_rcu_qs();
  16316. + return;
  16317. + }
  16318. + local_irq_enable();
  16319. +}
  16320. +
  16321. /*
  16322. * preempt_count and SOFTIRQ_OFFSET usage:
  16323. * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  16324. @@ -232,10 +403,8 @@
  16325. unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
  16326. unsigned long old_flags = current->flags;
  16327. int max_restart = MAX_SOFTIRQ_RESTART;
  16328. - struct softirq_action *h;
  16329. bool in_hardirq;
  16330. __u32 pending;
  16331. - int softirq_bit;
  16332. /*
  16333. * Mask out PF_MEMALLOC s current task context is borrowed for the
  16334. @@ -254,36 +423,7 @@
  16335. /* Reset the pending bitmask before enabling irqs */
  16336. set_softirq_pending(0);
  16337. - local_irq_enable();
  16338. -
  16339. - h = softirq_vec;
  16340. -
  16341. - while ((softirq_bit = ffs(pending))) {
  16342. - unsigned int vec_nr;
  16343. - int prev_count;
  16344. -
  16345. - h += softirq_bit - 1;
  16346. -
  16347. - vec_nr = h - softirq_vec;
  16348. - prev_count = preempt_count();
  16349. -
  16350. - kstat_incr_softirqs_this_cpu(vec_nr);
  16351. -
  16352. - trace_softirq_entry(vec_nr);
  16353. - h->action(h);
  16354. - trace_softirq_exit(vec_nr);
  16355. - if (unlikely(prev_count != preempt_count())) {
  16356. - pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
  16357. - vec_nr, softirq_to_name[vec_nr], h->action,
  16358. - prev_count, preempt_count());
  16359. - preempt_count_set(prev_count);
  16360. - }
  16361. - h++;
  16362. - pending >>= softirq_bit;
  16363. - }
  16364. -
  16365. - rcu_bh_qs();
  16366. - local_irq_disable();
  16367. + handle_pending_softirqs(pending);
  16368. pending = local_softirq_pending();
  16369. if (pending) {
  16370. @@ -320,6 +460,310 @@
  16371. }
  16372. /*
  16373. + * This function must run with irqs disabled!
  16374. + */
  16375. +void raise_softirq_irqoff(unsigned int nr)
  16376. +{
  16377. + __raise_softirq_irqoff(nr);
  16378. +
  16379. + /*
  16380. + * If we're in an interrupt or softirq, we're done
  16381. + * (this also catches softirq-disabled code). We will
  16382. + * actually run the softirq once we return from
  16383. + * the irq or softirq.
  16384. + *
  16385. + * Otherwise we wake up ksoftirqd to make sure we
  16386. + * schedule the softirq soon.
  16387. + */
  16388. + if (!in_interrupt())
  16389. + wakeup_softirqd();
  16390. +}
  16391. +
  16392. +void __raise_softirq_irqoff(unsigned int nr)
  16393. +{
  16394. + trace_softirq_raise(nr);
  16395. + or_softirq_pending(1UL << nr);
  16396. +}
  16397. +
  16398. +static inline void local_bh_disable_nort(void) { local_bh_disable(); }
  16399. +static inline void _local_bh_enable_nort(void) { _local_bh_enable(); }
  16400. +static void ksoftirqd_set_sched_params(unsigned int cpu) { }
  16401. +
  16402. +#else /* !PREEMPT_RT_FULL */
  16403. +
  16404. +/*
  16405. + * On RT we serialize softirq execution with a cpu local lock per softirq
  16406. + */
  16407. +static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks);
  16408. +
  16409. +void __init softirq_early_init(void)
  16410. +{
  16411. + int i;
  16412. +
  16413. + for (i = 0; i < NR_SOFTIRQS; i++)
  16414. + local_irq_lock_init(local_softirq_locks[i]);
  16415. +}
  16416. +
  16417. +static void lock_softirq(int which)
  16418. +{
  16419. + local_lock(local_softirq_locks[which]);
  16420. +}
  16421. +
  16422. +static void unlock_softirq(int which)
  16423. +{
  16424. + local_unlock(local_softirq_locks[which]);
  16425. +}
  16426. +
  16427. +static void do_single_softirq(int which)
  16428. +{
  16429. + unsigned long old_flags = current->flags;
  16430. +
  16431. + current->flags &= ~PF_MEMALLOC;
  16432. + vtime_account_irq_enter(current);
  16433. + current->flags |= PF_IN_SOFTIRQ;
  16434. + lockdep_softirq_enter();
  16435. + local_irq_enable();
  16436. + handle_softirq(which);
  16437. + local_irq_disable();
  16438. + lockdep_softirq_exit();
  16439. + current->flags &= ~PF_IN_SOFTIRQ;
  16440. + vtime_account_irq_enter(current);
  16441. + tsk_restore_flags(current, old_flags, PF_MEMALLOC);
  16442. +}
  16443. +
  16444. +/*
  16445. + * Called with interrupts disabled. Process softirqs which were raised
  16446. + * in current context (or on behalf of ksoftirqd).
  16447. + */
  16448. +static void do_current_softirqs(void)
  16449. +{
  16450. + while (current->softirqs_raised) {
  16451. + int i = __ffs(current->softirqs_raised);
  16452. + unsigned int pending, mask = (1U << i);
  16453. +
  16454. + current->softirqs_raised &= ~mask;
  16455. + local_irq_enable();
  16456. +
  16457. + /*
  16458. + * If the lock is contended, we boost the owner to
  16459. + * process the softirq or leave the critical section
  16460. + * now.
  16461. + */
  16462. + lock_softirq(i);
  16463. + local_irq_disable();
  16464. + softirq_set_runner(i);
  16465. + /*
  16466. + * Check with the local_softirq_pending() bits,
  16467. + * whether we need to process this still or if someone
  16468. + * else took care of it.
  16469. + */
  16470. + pending = local_softirq_pending();
  16471. + if (pending & mask) {
  16472. + set_softirq_pending(pending & ~mask);
  16473. + do_single_softirq(i);
  16474. + }
  16475. + softirq_clr_runner(i);
  16476. + WARN_ON(current->softirq_nestcnt != 1);
  16477. + local_irq_enable();
  16478. + unlock_softirq(i);
  16479. + local_irq_disable();
  16480. + }
  16481. +}
  16482. +
  16483. +void __local_bh_disable(void)
  16484. +{
  16485. + if (++current->softirq_nestcnt == 1)
  16486. + migrate_disable();
  16487. +}
  16488. +EXPORT_SYMBOL(__local_bh_disable);
  16489. +
  16490. +void __local_bh_enable(void)
  16491. +{
  16492. + if (WARN_ON(current->softirq_nestcnt == 0))
  16493. + return;
  16494. +
  16495. + local_irq_disable();
  16496. + if (current->softirq_nestcnt == 1 && current->softirqs_raised)
  16497. + do_current_softirqs();
  16498. + local_irq_enable();
  16499. +
  16500. + if (--current->softirq_nestcnt == 0)
  16501. + migrate_enable();
  16502. +}
  16503. +EXPORT_SYMBOL(__local_bh_enable);
  16504. +
  16505. +void _local_bh_enable(void)
  16506. +{
  16507. + if (WARN_ON(current->softirq_nestcnt == 0))
  16508. + return;
  16509. + if (--current->softirq_nestcnt == 0)
  16510. + migrate_enable();
  16511. +}
  16512. +EXPORT_SYMBOL(_local_bh_enable);
  16513. +
  16514. +int in_serving_softirq(void)
  16515. +{
  16516. + return current->flags & PF_IN_SOFTIRQ;
  16517. +}
  16518. +EXPORT_SYMBOL(in_serving_softirq);
  16519. +
  16520. +/* Called with preemption disabled */
  16521. +static void run_ksoftirqd(unsigned int cpu)
  16522. +{
  16523. + local_irq_disable();
  16524. + current->softirq_nestcnt++;
  16525. +
  16526. + do_current_softirqs();
  16527. + current->softirq_nestcnt--;
  16528. + local_irq_enable();
  16529. + cond_resched_rcu_qs();
  16530. +}
  16531. +
  16532. +/*
  16533. + * Called from netif_rx_ni(). Preemption enabled, but migration
  16534. + * disabled. So the cpu can't go away under us.
  16535. + */
  16536. +void thread_do_softirq(void)
  16537. +{
  16538. + if (!in_serving_softirq() && current->softirqs_raised) {
  16539. + current->softirq_nestcnt++;
  16540. + do_current_softirqs();
  16541. + current->softirq_nestcnt--;
  16542. + }
  16543. +}
  16544. +
  16545. +static void do_raise_softirq_irqoff(unsigned int nr)
  16546. +{
  16547. + unsigned int mask;
  16548. +
  16549. + mask = 1UL << nr;
  16550. +
  16551. + trace_softirq_raise(nr);
  16552. + or_softirq_pending(mask);
  16553. +
  16554. + /*
  16555. + * If we are not in a hard interrupt and inside a bh disabled
  16556. + * region, we simply raise the flag on current. local_bh_enable()
  16557. + * will make sure that the softirq is executed. Otherwise we
  16558. + * delegate it to ksoftirqd.
  16559. + */
  16560. + if (!in_irq() && current->softirq_nestcnt)
  16561. + current->softirqs_raised |= mask;
  16562. + else if (!__this_cpu_read(ksoftirqd) || !__this_cpu_read(ktimer_softirqd))
  16563. + return;
  16564. +
  16565. + if (mask & TIMER_SOFTIRQS)
  16566. + __this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
  16567. + else
  16568. + __this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
  16569. +}
  16570. +
  16571. +static void wakeup_proper_softirq(unsigned int nr)
  16572. +{
  16573. + if ((1UL << nr) & TIMER_SOFTIRQS)
  16574. + wakeup_timer_softirqd();
  16575. + else
  16576. + wakeup_softirqd();
  16577. +}
  16578. +
  16579. +
  16580. +void __raise_softirq_irqoff(unsigned int nr)
  16581. +{
  16582. + do_raise_softirq_irqoff(nr);
  16583. + if (!in_irq() && !current->softirq_nestcnt)
  16584. + wakeup_proper_softirq(nr);
  16585. +}
  16586. +
  16587. +/*
  16588. + * Same as __raise_softirq_irqoff() but will process them in ksoftirqd
  16589. + */
  16590. +void __raise_softirq_irqoff_ksoft(unsigned int nr)
  16591. +{
  16592. + unsigned int mask;
  16593. +
  16594. + if (WARN_ON_ONCE(!__this_cpu_read(ksoftirqd) ||
  16595. + !__this_cpu_read(ktimer_softirqd)))
  16596. + return;
  16597. + mask = 1UL << nr;
  16598. +
  16599. + trace_softirq_raise(nr);
  16600. + or_softirq_pending(mask);
  16601. + if (mask & TIMER_SOFTIRQS)
  16602. + __this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
  16603. + else
  16604. + __this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
  16605. + wakeup_proper_softirq(nr);
  16606. +}
  16607. +
  16608. +/*
  16609. + * This function must run with irqs disabled!
  16610. + */
  16611. +void raise_softirq_irqoff(unsigned int nr)
  16612. +{
  16613. + do_raise_softirq_irqoff(nr);
  16614. +
  16615. + /*
  16616. + * If we're in an hard interrupt we let irq return code deal
  16617. + * with the wakeup of ksoftirqd.
  16618. + */
  16619. + if (in_irq())
  16620. + return;
  16621. + /*
  16622. + * If we are in thread context but outside of a bh disabled
  16623. + * region, we need to wake ksoftirqd as well.
  16624. + *
  16625. + * CHECKME: Some of the places which do that could be wrapped
  16626. + * into local_bh_disable/enable pairs. Though it's unclear
  16627. + * whether this is worth the effort. To find those places just
  16628. + * raise a WARN() if the condition is met.
  16629. + */
  16630. + if (!current->softirq_nestcnt)
  16631. + wakeup_proper_softirq(nr);
  16632. +}
  16633. +
  16634. +static inline int ksoftirqd_softirq_pending(void)
  16635. +{
  16636. + return current->softirqs_raised;
  16637. +}
  16638. +
  16639. +static inline void local_bh_disable_nort(void) { }
  16640. +static inline void _local_bh_enable_nort(void) { }
  16641. +
  16642. +static inline void ksoftirqd_set_sched_params(unsigned int cpu)
  16643. +{
  16644. + /* Take over all but timer pending softirqs when starting */
  16645. + local_irq_disable();
  16646. + current->softirqs_raised = local_softirq_pending() & ~TIMER_SOFTIRQS;
  16647. + local_irq_enable();
  16648. +}
  16649. +
  16650. +static inline void ktimer_softirqd_set_sched_params(unsigned int cpu)
  16651. +{
  16652. + struct sched_param param = { .sched_priority = 1 };
  16653. +
  16654. + sched_setscheduler(current, SCHED_FIFO, &param);
  16655. +
  16656. + /* Take over timer pending softirqs when starting */
  16657. + local_irq_disable();
  16658. + current->softirqs_raised = local_softirq_pending() & TIMER_SOFTIRQS;
  16659. + local_irq_enable();
  16660. +}
  16661. +
  16662. +static inline void ktimer_softirqd_clr_sched_params(unsigned int cpu,
  16663. + bool online)
  16664. +{
  16665. + struct sched_param param = { .sched_priority = 0 };
  16666. +
  16667. + sched_setscheduler(current, SCHED_NORMAL, &param);
  16668. +}
  16669. +
  16670. +static int ktimer_softirqd_should_run(unsigned int cpu)
  16671. +{
  16672. + return current->softirqs_raised;
  16673. +}
  16674. +
  16675. +#endif /* PREEMPT_RT_FULL */
  16676. +/*
  16677. * Enter an interrupt context.
  16678. */
  16679. void irq_enter(void)
  16680. @@ -330,9 +774,9 @@
  16681. * Prevent raise_softirq from needlessly waking up ksoftirqd
  16682. * here, as softirq will be serviced on return from interrupt.
  16683. */
  16684. - local_bh_disable();
  16685. + local_bh_disable_nort();
  16686. tick_irq_enter();
  16687. - _local_bh_enable();
  16688. + _local_bh_enable_nort();
  16689. }
  16690. __irq_enter();
  16691. @@ -340,6 +784,7 @@
  16692. static inline void invoke_softirq(void)
  16693. {
  16694. +#ifndef CONFIG_PREEMPT_RT_FULL
  16695. if (!force_irqthreads) {
  16696. #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
  16697. /*
  16698. @@ -359,6 +804,18 @@
  16699. } else {
  16700. wakeup_softirqd();
  16701. }
  16702. +#else /* PREEMPT_RT_FULL */
  16703. + unsigned long flags;
  16704. +
  16705. + local_irq_save(flags);
  16706. + if (__this_cpu_read(ksoftirqd) &&
  16707. + __this_cpu_read(ksoftirqd)->softirqs_raised)
  16708. + wakeup_softirqd();
  16709. + if (__this_cpu_read(ktimer_softirqd) &&
  16710. + __this_cpu_read(ktimer_softirqd)->softirqs_raised)
  16711. + wakeup_timer_softirqd();
  16712. + local_irq_restore(flags);
  16713. +#endif
  16714. }
  16715. static inline void tick_irq_exit(void)
  16716. @@ -395,26 +852,6 @@
  16717. trace_hardirq_exit(); /* must be last! */
  16718. }
  16719. -/*
  16720. - * This function must run with irqs disabled!
  16721. - */
  16722. -inline void raise_softirq_irqoff(unsigned int nr)
  16723. -{
  16724. - __raise_softirq_irqoff(nr);
  16725. -
  16726. - /*
  16727. - * If we're in an interrupt or softirq, we're done
  16728. - * (this also catches softirq-disabled code). We will
  16729. - * actually run the softirq once we return from
  16730. - * the irq or softirq.
  16731. - *
  16732. - * Otherwise we wake up ksoftirqd to make sure we
  16733. - * schedule the softirq soon.
  16734. - */
  16735. - if (!in_interrupt())
  16736. - wakeup_softirqd();
  16737. -}
  16738. -
  16739. void raise_softirq(unsigned int nr)
  16740. {
  16741. unsigned long flags;
  16742. @@ -424,12 +861,6 @@
  16743. local_irq_restore(flags);
  16744. }
  16745. -void __raise_softirq_irqoff(unsigned int nr)
  16746. -{
  16747. - trace_softirq_raise(nr);
  16748. - or_softirq_pending(1UL << nr);
  16749. -}
  16750. -
  16751. void open_softirq(int nr, void (*action)(struct softirq_action *))
  16752. {
  16753. softirq_vec[nr].action = action;
  16754. @@ -446,15 +877,45 @@
  16755. static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
  16756. static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
  16757. +static void inline
  16758. +__tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr)
  16759. +{
  16760. + if (tasklet_trylock(t)) {
  16761. +again:
  16762. + /* We may have been preempted before tasklet_trylock
  16763. + * and __tasklet_action may have already run.
  16764. + * So double check the sched bit while the takslet
  16765. + * is locked before adding it to the list.
  16766. + */
  16767. + if (test_bit(TASKLET_STATE_SCHED, &t->state)) {
  16768. + t->next = NULL;
  16769. + *head->tail = t;
  16770. + head->tail = &(t->next);
  16771. + raise_softirq_irqoff(nr);
  16772. + tasklet_unlock(t);
  16773. + } else {
  16774. + /* This is subtle. If we hit the corner case above
  16775. + * It is possible that we get preempted right here,
  16776. + * and another task has successfully called
  16777. + * tasklet_schedule(), then this function, and
  16778. + * failed on the trylock. Thus we must be sure
  16779. + * before releasing the tasklet lock, that the
  16780. + * SCHED_BIT is clear. Otherwise the tasklet
  16781. + * may get its SCHED_BIT set, but not added to the
  16782. + * list
  16783. + */
  16784. + if (!tasklet_tryunlock(t))
  16785. + goto again;
  16786. + }
  16787. + }
  16788. +}
  16789. +
  16790. void __tasklet_schedule(struct tasklet_struct *t)
  16791. {
  16792. unsigned long flags;
  16793. local_irq_save(flags);
  16794. - t->next = NULL;
  16795. - *__this_cpu_read(tasklet_vec.tail) = t;
  16796. - __this_cpu_write(tasklet_vec.tail, &(t->next));
  16797. - raise_softirq_irqoff(TASKLET_SOFTIRQ);
  16798. + __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
  16799. local_irq_restore(flags);
  16800. }
  16801. EXPORT_SYMBOL(__tasklet_schedule);
  16802. @@ -464,10 +925,7 @@
  16803. unsigned long flags;
  16804. local_irq_save(flags);
  16805. - t->next = NULL;
  16806. - *__this_cpu_read(tasklet_hi_vec.tail) = t;
  16807. - __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
  16808. - raise_softirq_irqoff(HI_SOFTIRQ);
  16809. + __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
  16810. local_irq_restore(flags);
  16811. }
  16812. EXPORT_SYMBOL(__tasklet_hi_schedule);
  16813. @@ -476,82 +934,122 @@
  16814. {
  16815. BUG_ON(!irqs_disabled());
  16816. - t->next = __this_cpu_read(tasklet_hi_vec.head);
  16817. - __this_cpu_write(tasklet_hi_vec.head, t);
  16818. - __raise_softirq_irqoff(HI_SOFTIRQ);
  16819. + __tasklet_hi_schedule(t);
  16820. }
  16821. EXPORT_SYMBOL(__tasklet_hi_schedule_first);
  16822. -static void tasklet_action(struct softirq_action *a)
  16823. +void tasklet_enable(struct tasklet_struct *t)
  16824. {
  16825. - struct tasklet_struct *list;
  16826. + if (!atomic_dec_and_test(&t->count))
  16827. + return;
  16828. + if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
  16829. + tasklet_schedule(t);
  16830. +}
  16831. +EXPORT_SYMBOL(tasklet_enable);
  16832. - local_irq_disable();
  16833. - list = __this_cpu_read(tasklet_vec.head);
  16834. - __this_cpu_write(tasklet_vec.head, NULL);
  16835. - __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
  16836. - local_irq_enable();
  16837. +static void __tasklet_action(struct softirq_action *a,
  16838. + struct tasklet_struct *list)
  16839. +{
  16840. + int loops = 1000000;
  16841. while (list) {
  16842. struct tasklet_struct *t = list;
  16843. list = list->next;
  16844. - if (tasklet_trylock(t)) {
  16845. - if (!atomic_read(&t->count)) {
  16846. - if (!test_and_clear_bit(TASKLET_STATE_SCHED,
  16847. - &t->state))
  16848. - BUG();
  16849. - t->func(t->data);
  16850. - tasklet_unlock(t);
  16851. - continue;
  16852. - }
  16853. - tasklet_unlock(t);
  16854. + /*
  16855. + * Should always succeed - after a tasklist got on the
  16856. + * list (after getting the SCHED bit set from 0 to 1),
  16857. + * nothing but the tasklet softirq it got queued to can
  16858. + * lock it:
  16859. + */
  16860. + if (!tasklet_trylock(t)) {
  16861. + WARN_ON(1);
  16862. + continue;
  16863. }
  16864. - local_irq_disable();
  16865. t->next = NULL;
  16866. - *__this_cpu_read(tasklet_vec.tail) = t;
  16867. - __this_cpu_write(tasklet_vec.tail, &(t->next));
  16868. - __raise_softirq_irqoff(TASKLET_SOFTIRQ);
  16869. - local_irq_enable();
  16870. +
  16871. + /*
  16872. + * If we cannot handle the tasklet because it's disabled,
  16873. + * mark it as pending. tasklet_enable() will later
  16874. + * re-schedule the tasklet.
  16875. + */
  16876. + if (unlikely(atomic_read(&t->count))) {
  16877. +out_disabled:
  16878. + /* implicit unlock: */
  16879. + wmb();
  16880. + t->state = TASKLET_STATEF_PENDING;
  16881. + continue;
  16882. + }
  16883. +
  16884. + /*
  16885. + * After this point on the tasklet might be rescheduled
  16886. + * on another CPU, but it can only be added to another
  16887. + * CPU's tasklet list if we unlock the tasklet (which we
  16888. + * dont do yet).
  16889. + */
  16890. + if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
  16891. + WARN_ON(1);
  16892. +
  16893. +again:
  16894. + t->func(t->data);
  16895. +
  16896. + /*
  16897. + * Try to unlock the tasklet. We must use cmpxchg, because
  16898. + * another CPU might have scheduled or disabled the tasklet.
  16899. + * We only allow the STATE_RUN -> 0 transition here.
  16900. + */
  16901. + while (!tasklet_tryunlock(t)) {
  16902. + /*
  16903. + * If it got disabled meanwhile, bail out:
  16904. + */
  16905. + if (atomic_read(&t->count))
  16906. + goto out_disabled;
  16907. + /*
  16908. + * If it got scheduled meanwhile, re-execute
  16909. + * the tasklet function:
  16910. + */
  16911. + if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
  16912. + goto again;
  16913. + if (!--loops) {
  16914. + printk("hm, tasklet state: %08lx\n", t->state);
  16915. + WARN_ON(1);
  16916. + tasklet_unlock(t);
  16917. + break;
  16918. + }
  16919. + }
  16920. }
  16921. }
  16922. +static void tasklet_action(struct softirq_action *a)
  16923. +{
  16924. + struct tasklet_struct *list;
  16925. +
  16926. + local_irq_disable();
  16927. +
  16928. + list = __this_cpu_read(tasklet_vec.head);
  16929. + __this_cpu_write(tasklet_vec.head, NULL);
  16930. + __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
  16931. +
  16932. + local_irq_enable();
  16933. +
  16934. + __tasklet_action(a, list);
  16935. +}
  16936. +
  16937. static void tasklet_hi_action(struct softirq_action *a)
  16938. {
  16939. struct tasklet_struct *list;
  16940. local_irq_disable();
  16941. +
  16942. list = __this_cpu_read(tasklet_hi_vec.head);
  16943. __this_cpu_write(tasklet_hi_vec.head, NULL);
  16944. __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head));
  16945. - local_irq_enable();
  16946. - while (list) {
  16947. - struct tasklet_struct *t = list;
  16948. -
  16949. - list = list->next;
  16950. -
  16951. - if (tasklet_trylock(t)) {
  16952. - if (!atomic_read(&t->count)) {
  16953. - if (!test_and_clear_bit(TASKLET_STATE_SCHED,
  16954. - &t->state))
  16955. - BUG();
  16956. - t->func(t->data);
  16957. - tasklet_unlock(t);
  16958. - continue;
  16959. - }
  16960. - tasklet_unlock(t);
  16961. - }
  16962. + local_irq_enable();
  16963. - local_irq_disable();
  16964. - t->next = NULL;
  16965. - *__this_cpu_read(tasklet_hi_vec.tail) = t;
  16966. - __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
  16967. - __raise_softirq_irqoff(HI_SOFTIRQ);
  16968. - local_irq_enable();
  16969. - }
  16970. + __tasklet_action(a, list);
  16971. }
  16972. void tasklet_init(struct tasklet_struct *t,
  16973. @@ -572,7 +1070,7 @@
  16974. while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
  16975. do {
  16976. - yield();
  16977. + msleep(1);
  16978. } while (test_bit(TASKLET_STATE_SCHED, &t->state));
  16979. }
  16980. tasklet_unlock_wait(t);
  16981. @@ -646,25 +1144,26 @@
  16982. open_softirq(HI_SOFTIRQ, tasklet_hi_action);
  16983. }
  16984. -static int ksoftirqd_should_run(unsigned int cpu)
  16985. -{
  16986. - return local_softirq_pending();
  16987. -}
  16988. -
  16989. -static void run_ksoftirqd(unsigned int cpu)
  16990. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  16991. +void tasklet_unlock_wait(struct tasklet_struct *t)
  16992. {
  16993. - local_irq_disable();
  16994. - if (local_softirq_pending()) {
  16995. + while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
  16996. /*
  16997. - * We can safely run softirq on inline stack, as we are not deep
  16998. - * in the task stack here.
  16999. + * Hack for now to avoid this busy-loop:
  17000. */
  17001. - __do_softirq();
  17002. - local_irq_enable();
  17003. - cond_resched_rcu_qs();
  17004. - return;
  17005. +#ifdef CONFIG_PREEMPT_RT_FULL
  17006. + msleep(1);
  17007. +#else
  17008. + barrier();
  17009. +#endif
  17010. }
  17011. - local_irq_enable();
  17012. +}
  17013. +EXPORT_SYMBOL(tasklet_unlock_wait);
  17014. +#endif
  17015. +
  17016. +static int ksoftirqd_should_run(unsigned int cpu)
  17017. +{
  17018. + return ksoftirqd_softirq_pending();
  17019. }
  17020. #ifdef CONFIG_HOTPLUG_CPU
  17021. @@ -746,16 +1245,31 @@
  17022. static struct smp_hotplug_thread softirq_threads = {
  17023. .store = &ksoftirqd,
  17024. + .setup = ksoftirqd_set_sched_params,
  17025. .thread_should_run = ksoftirqd_should_run,
  17026. .thread_fn = run_ksoftirqd,
  17027. .thread_comm = "ksoftirqd/%u",
  17028. };
  17029. +#ifdef CONFIG_PREEMPT_RT_FULL
  17030. +static struct smp_hotplug_thread softirq_timer_threads = {
  17031. + .store = &ktimer_softirqd,
  17032. + .setup = ktimer_softirqd_set_sched_params,
  17033. + .cleanup = ktimer_softirqd_clr_sched_params,
  17034. + .thread_should_run = ktimer_softirqd_should_run,
  17035. + .thread_fn = run_ksoftirqd,
  17036. + .thread_comm = "ktimersoftd/%u",
  17037. +};
  17038. +#endif
  17039. +
  17040. static __init int spawn_ksoftirqd(void)
  17041. {
  17042. register_cpu_notifier(&cpu_nfb);
  17043. BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
  17044. +#ifdef CONFIG_PREEMPT_RT_FULL
  17045. + BUG_ON(smpboot_register_percpu_thread(&softirq_timer_threads));
  17046. +#endif
  17047. return 0;
  17048. }
  17049. diff -Nur linux-4.6.6.orig/kernel/stop_machine.c linux-4.6.6/kernel/stop_machine.c
  17050. --- linux-4.6.6.orig/kernel/stop_machine.c 2016-08-10 12:54:57.000000000 +0200
  17051. +++ linux-4.6.6/kernel/stop_machine.c 2016-08-22 21:27:47.417893368 +0200
  17052. @@ -36,7 +36,7 @@
  17053. struct cpu_stopper {
  17054. struct task_struct *thread;
  17055. - spinlock_t lock;
  17056. + raw_spinlock_t lock;
  17057. bool enabled; /* is this stopper enabled? */
  17058. struct list_head works; /* list of pending works */
  17059. @@ -82,14 +82,14 @@
  17060. unsigned long flags;
  17061. bool enabled;
  17062. - spin_lock_irqsave(&stopper->lock, flags);
  17063. + raw_spin_lock_irqsave(&stopper->lock, flags);
  17064. enabled = stopper->enabled;
  17065. if (enabled)
  17066. __cpu_stop_queue_work(stopper, work);
  17067. else if (work->done)
  17068. cpu_stop_signal_done(work->done);
  17069. - spin_unlock_irqrestore(&stopper->lock, flags);
  17070. + raw_spin_unlock_irqrestore(&stopper->lock, flags);
  17071. return enabled;
  17072. }
  17073. @@ -224,8 +224,8 @@
  17074. int err;
  17075. lg_double_lock(&stop_cpus_lock, cpu1, cpu2);
  17076. - spin_lock_irq(&stopper1->lock);
  17077. - spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
  17078. + raw_spin_lock_irq(&stopper1->lock);
  17079. + raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
  17080. err = -ENOENT;
  17081. if (!stopper1->enabled || !stopper2->enabled)
  17082. @@ -235,8 +235,8 @@
  17083. __cpu_stop_queue_work(stopper1, work1);
  17084. __cpu_stop_queue_work(stopper2, work2);
  17085. unlock:
  17086. - spin_unlock(&stopper2->lock);
  17087. - spin_unlock_irq(&stopper1->lock);
  17088. + raw_spin_unlock(&stopper2->lock);
  17089. + raw_spin_unlock_irq(&stopper1->lock);
  17090. lg_double_unlock(&stop_cpus_lock, cpu1, cpu2);
  17091. return err;
  17092. @@ -313,18 +313,21 @@
  17093. static bool queue_stop_cpus_work(const struct cpumask *cpumask,
  17094. cpu_stop_fn_t fn, void *arg,
  17095. - struct cpu_stop_done *done)
  17096. + struct cpu_stop_done *done, bool inactive)
  17097. {
  17098. struct cpu_stop_work *work;
  17099. unsigned int cpu;
  17100. bool queued = false;
  17101. /*
  17102. - * Disable preemption while queueing to avoid getting
  17103. - * preempted by a stopper which might wait for other stoppers
  17104. - * to enter @fn which can lead to deadlock.
  17105. + * Make sure that all work is queued on all cpus before
  17106. + * any of the cpus can execute it.
  17107. */
  17108. - lg_global_lock(&stop_cpus_lock);
  17109. + if (!inactive)
  17110. + lg_global_lock(&stop_cpus_lock);
  17111. + else
  17112. + lg_global_trylock_relax(&stop_cpus_lock);
  17113. +
  17114. for_each_cpu(cpu, cpumask) {
  17115. work = &per_cpu(cpu_stopper.stop_work, cpu);
  17116. work->fn = fn;
  17117. @@ -344,7 +347,7 @@
  17118. struct cpu_stop_done done;
  17119. cpu_stop_init_done(&done, cpumask_weight(cpumask));
  17120. - if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
  17121. + if (!queue_stop_cpus_work(cpumask, fn, arg, &done, false))
  17122. return -ENOENT;
  17123. wait_for_completion(&done.completion);
  17124. return done.ret;
  17125. @@ -425,9 +428,9 @@
  17126. unsigned long flags;
  17127. int run;
  17128. - spin_lock_irqsave(&stopper->lock, flags);
  17129. + raw_spin_lock_irqsave(&stopper->lock, flags);
  17130. run = !list_empty(&stopper->works);
  17131. - spin_unlock_irqrestore(&stopper->lock, flags);
  17132. + raw_spin_unlock_irqrestore(&stopper->lock, flags);
  17133. return run;
  17134. }
  17135. @@ -438,13 +441,13 @@
  17136. repeat:
  17137. work = NULL;
  17138. - spin_lock_irq(&stopper->lock);
  17139. + raw_spin_lock_irq(&stopper->lock);
  17140. if (!list_empty(&stopper->works)) {
  17141. work = list_first_entry(&stopper->works,
  17142. struct cpu_stop_work, list);
  17143. list_del_init(&work->list);
  17144. }
  17145. - spin_unlock_irq(&stopper->lock);
  17146. + raw_spin_unlock_irq(&stopper->lock);
  17147. if (work) {
  17148. cpu_stop_fn_t fn = work->fn;
  17149. @@ -452,6 +455,16 @@
  17150. struct cpu_stop_done *done = work->done;
  17151. int ret;
  17152. + /*
  17153. + * Wait until the stopper finished scheduling on all
  17154. + * cpus
  17155. + */
  17156. + lg_global_lock(&stop_cpus_lock);
  17157. + /*
  17158. + * Let other cpu threads continue as well
  17159. + */
  17160. + lg_global_unlock(&stop_cpus_lock);
  17161. +
  17162. /* cpu stop callbacks must not sleep, make in_atomic() == T */
  17163. preempt_count_inc();
  17164. ret = fn(arg);
  17165. @@ -518,10 +531,12 @@
  17166. for_each_possible_cpu(cpu) {
  17167. struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
  17168. - spin_lock_init(&stopper->lock);
  17169. + raw_spin_lock_init(&stopper->lock);
  17170. INIT_LIST_HEAD(&stopper->works);
  17171. }
  17172. + lg_lock_init(&stop_cpus_lock, "stop_cpus_lock");
  17173. +
  17174. BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
  17175. stop_machine_unpark(raw_smp_processor_id());
  17176. stop_machine_initialized = true;
  17177. @@ -616,7 +631,7 @@
  17178. set_state(&msdata, MULTI_STOP_PREPARE);
  17179. cpu_stop_init_done(&done, num_active_cpus());
  17180. queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
  17181. - &done);
  17182. + &done, true);
  17183. ret = multi_cpu_stop(&msdata);
  17184. /* Busy wait for completion. */
  17185. diff -Nur linux-4.6.6.orig/kernel/time/hrtimer.c linux-4.6.6/kernel/time/hrtimer.c
  17186. --- linux-4.6.6.orig/kernel/time/hrtimer.c 2016-08-10 12:54:57.000000000 +0200
  17187. +++ linux-4.6.6/kernel/time/hrtimer.c 2016-08-22 21:27:47.421893628 +0200
  17188. @@ -48,11 +48,13 @@
  17189. #include <linux/sched/rt.h>
  17190. #include <linux/sched/deadline.h>
  17191. #include <linux/timer.h>
  17192. +#include <linux/kthread.h>
  17193. #include <linux/freezer.h>
  17194. #include <asm/uaccess.h>
  17195. #include <trace/events/timer.h>
  17196. +#include <trace/events/hist.h>
  17197. #include "tick-internal.h"
  17198. @@ -706,6 +708,44 @@
  17199. static DECLARE_WORK(hrtimer_work, clock_was_set_work);
  17200. +#ifdef CONFIG_PREEMPT_RT_FULL
  17201. +/*
  17202. + * RT can not call schedule_work from real interrupt context.
  17203. + * Need to make a thread to do the real work.
  17204. + */
  17205. +static struct task_struct *clock_set_delay_thread;
  17206. +static bool do_clock_set_delay;
  17207. +
  17208. +static int run_clock_set_delay(void *ignore)
  17209. +{
  17210. + while (!kthread_should_stop()) {
  17211. + set_current_state(TASK_INTERRUPTIBLE);
  17212. + if (do_clock_set_delay) {
  17213. + do_clock_set_delay = false;
  17214. + schedule_work(&hrtimer_work);
  17215. + }
  17216. + schedule();
  17217. + }
  17218. + __set_current_state(TASK_RUNNING);
  17219. + return 0;
  17220. +}
  17221. +
  17222. +void clock_was_set_delayed(void)
  17223. +{
  17224. + do_clock_set_delay = true;
  17225. + /* Make visible before waking up process */
  17226. + smp_wmb();
  17227. + wake_up_process(clock_set_delay_thread);
  17228. +}
  17229. +
  17230. +static __init int create_clock_set_delay_thread(void)
  17231. +{
  17232. + clock_set_delay_thread = kthread_run(run_clock_set_delay, NULL, "kclksetdelayd");
  17233. + BUG_ON(!clock_set_delay_thread);
  17234. + return 0;
  17235. +}
  17236. +early_initcall(create_clock_set_delay_thread);
  17237. +#else /* PREEMPT_RT_FULL */
  17238. /*
  17239. * Called from timekeeping and resume code to reprogramm the hrtimer
  17240. * interrupt device on all cpus.
  17241. @@ -714,6 +754,7 @@
  17242. {
  17243. schedule_work(&hrtimer_work);
  17244. }
  17245. +#endif
  17246. #else
  17247. @@ -723,11 +764,8 @@
  17248. static inline void hrtimer_switch_to_hres(void) { }
  17249. static inline void
  17250. hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
  17251. -static inline int hrtimer_reprogram(struct hrtimer *timer,
  17252. - struct hrtimer_clock_base *base)
  17253. -{
  17254. - return 0;
  17255. -}
  17256. +static inline void hrtimer_reprogram(struct hrtimer *timer,
  17257. + struct hrtimer_clock_base *base) { }
  17258. static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
  17259. static inline void retrigger_next_event(void *arg) { }
  17260. @@ -859,6 +897,32 @@
  17261. }
  17262. EXPORT_SYMBOL_GPL(hrtimer_forward);
  17263. +#ifdef CONFIG_PREEMPT_RT_BASE
  17264. +# define wake_up_timer_waiters(b) wake_up(&(b)->wait)
  17265. +
  17266. +/**
  17267. + * hrtimer_wait_for_timer - Wait for a running timer
  17268. + *
  17269. + * @timer: timer to wait for
  17270. + *
  17271. + * The function waits in case the timers callback function is
  17272. + * currently executed on the waitqueue of the timer base. The
  17273. + * waitqueue is woken up after the timer callback function has
  17274. + * finished execution.
  17275. + */
  17276. +void hrtimer_wait_for_timer(const struct hrtimer *timer)
  17277. +{
  17278. + struct hrtimer_clock_base *base = timer->base;
  17279. +
  17280. + if (base && base->cpu_base && !timer->irqsafe)
  17281. + wait_event(base->cpu_base->wait,
  17282. + !(hrtimer_callback_running(timer)));
  17283. +}
  17284. +
  17285. +#else
  17286. +# define wake_up_timer_waiters(b) do { } while (0)
  17287. +#endif
  17288. +
  17289. /*
  17290. * enqueue_hrtimer - internal function to (re)start a timer
  17291. *
  17292. @@ -900,6 +964,11 @@
  17293. if (!(state & HRTIMER_STATE_ENQUEUED))
  17294. return;
  17295. + if (unlikely(!list_empty(&timer->cb_entry))) {
  17296. + list_del_init(&timer->cb_entry);
  17297. + return;
  17298. + }
  17299. +
  17300. if (!timerqueue_del(&base->active, &timer->node))
  17301. cpu_base->active_bases &= ~(1 << base->index);
  17302. @@ -995,7 +1064,16 @@
  17303. new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
  17304. timer_stats_hrtimer_set_start_info(timer);
  17305. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  17306. + {
  17307. + ktime_t now = new_base->get_time();
  17308. + if (ktime_to_ns(tim) < ktime_to_ns(now))
  17309. + timer->praecox = now;
  17310. + else
  17311. + timer->praecox = ktime_set(0, 0);
  17312. + }
  17313. +#endif
  17314. leftmost = enqueue_hrtimer(timer, new_base);
  17315. if (!leftmost)
  17316. goto unlock;
  17317. @@ -1067,7 +1145,7 @@
  17318. if (ret >= 0)
  17319. return ret;
  17320. - cpu_relax();
  17321. + hrtimer_wait_for_timer(timer);
  17322. }
  17323. }
  17324. EXPORT_SYMBOL_GPL(hrtimer_cancel);
  17325. @@ -1131,6 +1209,7 @@
  17326. base = hrtimer_clockid_to_base(clock_id);
  17327. timer->base = &cpu_base->clock_base[base];
  17328. + INIT_LIST_HEAD(&timer->cb_entry);
  17329. timerqueue_init(&timer->node);
  17330. #ifdef CONFIG_TIMER_STATS
  17331. @@ -1171,6 +1250,7 @@
  17332. seq = raw_read_seqcount_begin(&cpu_base->seq);
  17333. if (timer->state != HRTIMER_STATE_INACTIVE ||
  17334. + cpu_base->running_soft == timer ||
  17335. cpu_base->running == timer)
  17336. return true;
  17337. @@ -1269,10 +1349,112 @@
  17338. cpu_base->running = NULL;
  17339. }
  17340. +#ifdef CONFIG_PREEMPT_RT_BASE
  17341. +static void hrtimer_rt_reprogram(int restart, struct hrtimer *timer,
  17342. + struct hrtimer_clock_base *base)
  17343. +{
  17344. + int leftmost;
  17345. +
  17346. + if (restart != HRTIMER_NORESTART &&
  17347. + !(timer->state & HRTIMER_STATE_ENQUEUED)) {
  17348. +
  17349. + leftmost = enqueue_hrtimer(timer, base);
  17350. + if (!leftmost)
  17351. + return;
  17352. +#ifdef CONFIG_HIGH_RES_TIMERS
  17353. + if (!hrtimer_is_hres_active(timer)) {
  17354. + /*
  17355. + * Kick to reschedule the next tick to handle the new timer
  17356. + * on dynticks target.
  17357. + */
  17358. + if (base->cpu_base->nohz_active)
  17359. + wake_up_nohz_cpu(base->cpu_base->cpu);
  17360. + } else {
  17361. +
  17362. + hrtimer_reprogram(timer, base);
  17363. + }
  17364. +#endif
  17365. + }
  17366. +}
  17367. +
  17368. +/*
  17369. + * The changes in mainline which removed the callback modes from
  17370. + * hrtimer are not yet working with -rt. The non wakeup_process()
  17371. + * based callbacks which involve sleeping locks need to be treated
  17372. + * seperately.
  17373. + */
  17374. +static void hrtimer_rt_run_pending(void)
  17375. +{
  17376. + enum hrtimer_restart (*fn)(struct hrtimer *);
  17377. + struct hrtimer_cpu_base *cpu_base;
  17378. + struct hrtimer_clock_base *base;
  17379. + struct hrtimer *timer;
  17380. + int index, restart;
  17381. +
  17382. + local_irq_disable();
  17383. + cpu_base = &per_cpu(hrtimer_bases, smp_processor_id());
  17384. +
  17385. + raw_spin_lock(&cpu_base->lock);
  17386. +
  17387. + for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) {
  17388. + base = &cpu_base->clock_base[index];
  17389. +
  17390. + while (!list_empty(&base->expired)) {
  17391. + timer = list_first_entry(&base->expired,
  17392. + struct hrtimer, cb_entry);
  17393. +
  17394. + /*
  17395. + * Same as the above __run_hrtimer function
  17396. + * just we run with interrupts enabled.
  17397. + */
  17398. + debug_deactivate(timer);
  17399. + cpu_base->running_soft = timer;
  17400. + raw_write_seqcount_barrier(&cpu_base->seq);
  17401. +
  17402. + __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
  17403. + timer_stats_account_hrtimer(timer);
  17404. + fn = timer->function;
  17405. +
  17406. + raw_spin_unlock_irq(&cpu_base->lock);
  17407. + restart = fn(timer);
  17408. + raw_spin_lock_irq(&cpu_base->lock);
  17409. +
  17410. + hrtimer_rt_reprogram(restart, timer, base);
  17411. + raw_write_seqcount_barrier(&cpu_base->seq);
  17412. +
  17413. + WARN_ON_ONCE(cpu_base->running_soft != timer);
  17414. + cpu_base->running_soft = NULL;
  17415. + }
  17416. + }
  17417. +
  17418. + raw_spin_unlock_irq(&cpu_base->lock);
  17419. +
  17420. + wake_up_timer_waiters(cpu_base);
  17421. +}
  17422. +
  17423. +static int hrtimer_rt_defer(struct hrtimer *timer)
  17424. +{
  17425. + if (timer->irqsafe)
  17426. + return 0;
  17427. +
  17428. + __remove_hrtimer(timer, timer->base, timer->state, 0);
  17429. + list_add_tail(&timer->cb_entry, &timer->base->expired);
  17430. + return 1;
  17431. +}
  17432. +
  17433. +#else
  17434. +
  17435. +static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; }
  17436. +
  17437. +#endif
  17438. +
  17439. +static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer);
  17440. +
  17441. static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
  17442. {
  17443. struct hrtimer_clock_base *base = cpu_base->clock_base;
  17444. unsigned int active = cpu_base->active_bases;
  17445. + int raise = 0;
  17446. for (; active; base++, active >>= 1) {
  17447. struct timerqueue_node *node;
  17448. @@ -1288,6 +1470,15 @@
  17449. timer = container_of(node, struct hrtimer, node);
  17450. + trace_hrtimer_interrupt(raw_smp_processor_id(),
  17451. + ktime_to_ns(ktime_sub(ktime_to_ns(timer->praecox) ?
  17452. + timer->praecox : hrtimer_get_expires(timer),
  17453. + basenow)),
  17454. + current,
  17455. + timer->function == hrtimer_wakeup ?
  17456. + container_of(timer, struct hrtimer_sleeper,
  17457. + timer)->task : NULL);
  17458. +
  17459. /*
  17460. * The immediate goal for using the softexpires is
  17461. * minimizing wakeups, not running timers at the
  17462. @@ -1303,9 +1494,14 @@
  17463. if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer))
  17464. break;
  17465. - __run_hrtimer(cpu_base, base, timer, &basenow);
  17466. + if (!hrtimer_rt_defer(timer))
  17467. + __run_hrtimer(cpu_base, base, timer, &basenow);
  17468. + else
  17469. + raise = 1;
  17470. }
  17471. }
  17472. + if (raise)
  17473. + raise_softirq_irqoff(HRTIMER_SOFTIRQ);
  17474. }
  17475. #ifdef CONFIG_HIGH_RES_TIMERS
  17476. @@ -1468,16 +1664,18 @@
  17477. void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
  17478. {
  17479. sl->timer.function = hrtimer_wakeup;
  17480. + sl->timer.irqsafe = 1;
  17481. sl->task = task;
  17482. }
  17483. EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
  17484. -static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
  17485. +static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode,
  17486. + unsigned long state)
  17487. {
  17488. hrtimer_init_sleeper(t, current);
  17489. do {
  17490. - set_current_state(TASK_INTERRUPTIBLE);
  17491. + set_current_state(state);
  17492. hrtimer_start_expires(&t->timer, mode);
  17493. if (likely(t->task))
  17494. @@ -1519,7 +1717,8 @@
  17495. HRTIMER_MODE_ABS);
  17496. hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
  17497. - if (do_nanosleep(&t, HRTIMER_MODE_ABS))
  17498. + /* cpu_chill() does not care about restart state. */
  17499. + if (do_nanosleep(&t, HRTIMER_MODE_ABS, TASK_INTERRUPTIBLE))
  17500. goto out;
  17501. rmtp = restart->nanosleep.rmtp;
  17502. @@ -1536,8 +1735,10 @@
  17503. return ret;
  17504. }
  17505. -long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  17506. - const enum hrtimer_mode mode, const clockid_t clockid)
  17507. +static long
  17508. +__hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  17509. + const enum hrtimer_mode mode, const clockid_t clockid,
  17510. + unsigned long state)
  17511. {
  17512. struct restart_block *restart;
  17513. struct hrtimer_sleeper t;
  17514. @@ -1550,7 +1751,7 @@
  17515. hrtimer_init_on_stack(&t.timer, clockid, mode);
  17516. hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);
  17517. - if (do_nanosleep(&t, mode))
  17518. + if (do_nanosleep(&t, mode, state))
  17519. goto out;
  17520. /* Absolute timers do not update the rmtp value and restart: */
  17521. @@ -1577,6 +1778,12 @@
  17522. return ret;
  17523. }
  17524. +long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  17525. + const enum hrtimer_mode mode, const clockid_t clockid)
  17526. +{
  17527. + return __hrtimer_nanosleep(rqtp, rmtp, mode, clockid, TASK_INTERRUPTIBLE);
  17528. +}
  17529. +
  17530. SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp,
  17531. struct timespec __user *, rmtp)
  17532. {
  17533. @@ -1591,6 +1798,26 @@
  17534. return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
  17535. }
  17536. +#ifdef CONFIG_PREEMPT_RT_FULL
  17537. +/*
  17538. + * Sleep for 1 ms in hope whoever holds what we want will let it go.
  17539. + */
  17540. +void cpu_chill(void)
  17541. +{
  17542. + struct timespec tu = {
  17543. + .tv_nsec = NSEC_PER_MSEC,
  17544. + };
  17545. + unsigned int freeze_flag = current->flags & PF_NOFREEZE;
  17546. +
  17547. + current->flags |= PF_NOFREEZE;
  17548. + __hrtimer_nanosleep(&tu, NULL, HRTIMER_MODE_REL, CLOCK_MONOTONIC,
  17549. + TASK_UNINTERRUPTIBLE);
  17550. + if (!freeze_flag)
  17551. + current->flags &= ~PF_NOFREEZE;
  17552. +}
  17553. +EXPORT_SYMBOL(cpu_chill);
  17554. +#endif
  17555. +
  17556. /*
  17557. * Functions related to boot-time initialization:
  17558. */
  17559. @@ -1602,10 +1829,14 @@
  17560. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
  17561. cpu_base->clock_base[i].cpu_base = cpu_base;
  17562. timerqueue_init_head(&cpu_base->clock_base[i].active);
  17563. + INIT_LIST_HEAD(&cpu_base->clock_base[i].expired);
  17564. }
  17565. cpu_base->cpu = cpu;
  17566. hrtimer_init_hres(cpu_base);
  17567. +#ifdef CONFIG_PREEMPT_RT_BASE
  17568. + init_waitqueue_head(&cpu_base->wait);
  17569. +#endif
  17570. }
  17571. #ifdef CONFIG_HOTPLUG_CPU
  17572. @@ -1703,11 +1934,21 @@
  17573. .notifier_call = hrtimer_cpu_notify,
  17574. };
  17575. +#ifdef CONFIG_PREEMPT_RT_BASE
  17576. +static void run_hrtimer_softirq(struct softirq_action *h)
  17577. +{
  17578. + hrtimer_rt_run_pending();
  17579. +}
  17580. +#endif
  17581. +
  17582. void __init hrtimers_init(void)
  17583. {
  17584. hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
  17585. (void *)(long)smp_processor_id());
  17586. register_cpu_notifier(&hrtimers_nb);
  17587. +#ifdef CONFIG_PREEMPT_RT_BASE
  17588. + open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
  17589. +#endif
  17590. }
  17591. /**
  17592. diff -Nur linux-4.6.6.orig/kernel/time/itimer.c linux-4.6.6/kernel/time/itimer.c
  17593. --- linux-4.6.6.orig/kernel/time/itimer.c 2016-08-10 12:54:57.000000000 +0200
  17594. +++ linux-4.6.6/kernel/time/itimer.c 2016-08-22 21:27:47.421893628 +0200
  17595. @@ -213,6 +213,7 @@
  17596. /* We are sharing ->siglock with it_real_fn() */
  17597. if (hrtimer_try_to_cancel(timer) < 0) {
  17598. spin_unlock_irq(&tsk->sighand->siglock);
  17599. + hrtimer_wait_for_timer(&tsk->signal->real_timer);
  17600. goto again;
  17601. }
  17602. expires = timeval_to_ktime(value->it_value);
  17603. diff -Nur linux-4.6.6.orig/kernel/time/jiffies.c linux-4.6.6/kernel/time/jiffies.c
  17604. --- linux-4.6.6.orig/kernel/time/jiffies.c 2016-08-10 12:54:57.000000000 +0200
  17605. +++ linux-4.6.6/kernel/time/jiffies.c 2016-08-22 21:27:47.421893628 +0200
  17606. @@ -74,7 +74,8 @@
  17607. .max_cycles = 10,
  17608. };
  17609. -__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock);
  17610. +__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock);
  17611. +__cacheline_aligned_in_smp seqcount_t jiffies_seq;
  17612. #if (BITS_PER_LONG < 64)
  17613. u64 get_jiffies_64(void)
  17614. @@ -83,9 +84,9 @@
  17615. u64 ret;
  17616. do {
  17617. - seq = read_seqbegin(&jiffies_lock);
  17618. + seq = read_seqcount_begin(&jiffies_seq);
  17619. ret = jiffies_64;
  17620. - } while (read_seqretry(&jiffies_lock, seq));
  17621. + } while (read_seqcount_retry(&jiffies_seq, seq));
  17622. return ret;
  17623. }
  17624. EXPORT_SYMBOL(get_jiffies_64);
  17625. diff -Nur linux-4.6.6.orig/kernel/time/ntp.c linux-4.6.6/kernel/time/ntp.c
  17626. --- linux-4.6.6.orig/kernel/time/ntp.c 2016-08-10 12:54:57.000000000 +0200
  17627. +++ linux-4.6.6/kernel/time/ntp.c 2016-08-22 21:27:47.421893628 +0200
  17628. @@ -10,6 +10,7 @@
  17629. #include <linux/workqueue.h>
  17630. #include <linux/hrtimer.h>
  17631. #include <linux/jiffies.h>
  17632. +#include <linux/kthread.h>
  17633. #include <linux/math64.h>
  17634. #include <linux/timex.h>
  17635. #include <linux/time.h>
  17636. @@ -568,10 +569,52 @@
  17637. &sync_cmos_work, timespec64_to_jiffies(&next));
  17638. }
  17639. +#ifdef CONFIG_PREEMPT_RT_FULL
  17640. +/*
  17641. + * RT can not call schedule_delayed_work from real interrupt context.
  17642. + * Need to make a thread to do the real work.
  17643. + */
  17644. +static struct task_struct *cmos_delay_thread;
  17645. +static bool do_cmos_delay;
  17646. +
  17647. +static int run_cmos_delay(void *ignore)
  17648. +{
  17649. + while (!kthread_should_stop()) {
  17650. + set_current_state(TASK_INTERRUPTIBLE);
  17651. + if (do_cmos_delay) {
  17652. + do_cmos_delay = false;
  17653. + queue_delayed_work(system_power_efficient_wq,
  17654. + &sync_cmos_work, 0);
  17655. + }
  17656. + schedule();
  17657. + }
  17658. + __set_current_state(TASK_RUNNING);
  17659. + return 0;
  17660. +}
  17661. +
  17662. +void ntp_notify_cmos_timer(void)
  17663. +{
  17664. + do_cmos_delay = true;
  17665. + /* Make visible before waking up process */
  17666. + smp_wmb();
  17667. + wake_up_process(cmos_delay_thread);
  17668. +}
  17669. +
  17670. +static __init int create_cmos_delay_thread(void)
  17671. +{
  17672. + cmos_delay_thread = kthread_run(run_cmos_delay, NULL, "kcmosdelayd");
  17673. + BUG_ON(!cmos_delay_thread);
  17674. + return 0;
  17675. +}
  17676. +early_initcall(create_cmos_delay_thread);
  17677. +
  17678. +#else
  17679. +
  17680. void ntp_notify_cmos_timer(void)
  17681. {
  17682. queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0);
  17683. }
  17684. +#endif /* CONFIG_PREEMPT_RT_FULL */
  17685. #else
  17686. void ntp_notify_cmos_timer(void) { }
  17687. diff -Nur linux-4.6.6.orig/kernel/time/posix-cpu-timers.c linux-4.6.6/kernel/time/posix-cpu-timers.c
  17688. --- linux-4.6.6.orig/kernel/time/posix-cpu-timers.c 2016-08-10 12:54:57.000000000 +0200
  17689. +++ linux-4.6.6/kernel/time/posix-cpu-timers.c 2016-08-22 21:27:47.421893628 +0200
  17690. @@ -3,6 +3,7 @@
  17691. */
  17692. #include <linux/sched.h>
  17693. +#include <linux/sched/rt.h>
  17694. #include <linux/posix-timers.h>
  17695. #include <linux/errno.h>
  17696. #include <linux/math64.h>
  17697. @@ -620,7 +621,7 @@
  17698. /*
  17699. * Disarm any old timer after extracting its expiry time.
  17700. */
  17701. - WARN_ON_ONCE(!irqs_disabled());
  17702. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  17703. ret = 0;
  17704. old_incr = timer->it.cpu.incr;
  17705. @@ -1064,7 +1065,7 @@
  17706. /*
  17707. * Now re-arm for the new expiry time.
  17708. */
  17709. - WARN_ON_ONCE(!irqs_disabled());
  17710. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  17711. arm_timer(timer);
  17712. unlock_task_sighand(p, &flags);
  17713. @@ -1153,13 +1154,13 @@
  17714. * already updated our counts. We need to check if any timers fire now.
  17715. * Interrupts are disabled.
  17716. */
  17717. -void run_posix_cpu_timers(struct task_struct *tsk)
  17718. +static void __run_posix_cpu_timers(struct task_struct *tsk)
  17719. {
  17720. LIST_HEAD(firing);
  17721. struct k_itimer *timer, *next;
  17722. unsigned long flags;
  17723. - WARN_ON_ONCE(!irqs_disabled());
  17724. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  17725. /*
  17726. * The fast path checks that there are no expired thread or thread
  17727. @@ -1213,6 +1214,190 @@
  17728. }
  17729. }
  17730. +#ifdef CONFIG_PREEMPT_RT_BASE
  17731. +#include <linux/kthread.h>
  17732. +#include <linux/cpu.h>
  17733. +DEFINE_PER_CPU(struct task_struct *, posix_timer_task);
  17734. +DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist);
  17735. +
  17736. +static int posix_cpu_timers_thread(void *data)
  17737. +{
  17738. + int cpu = (long)data;
  17739. +
  17740. + BUG_ON(per_cpu(posix_timer_task,cpu) != current);
  17741. +
  17742. + while (!kthread_should_stop()) {
  17743. + struct task_struct *tsk = NULL;
  17744. + struct task_struct *next = NULL;
  17745. +
  17746. + if (cpu_is_offline(cpu))
  17747. + goto wait_to_die;
  17748. +
  17749. + /* grab task list */
  17750. + raw_local_irq_disable();
  17751. + tsk = per_cpu(posix_timer_tasklist, cpu);
  17752. + per_cpu(posix_timer_tasklist, cpu) = NULL;
  17753. + raw_local_irq_enable();
  17754. +
  17755. + /* its possible the list is empty, just return */
  17756. + if (!tsk) {
  17757. + set_current_state(TASK_INTERRUPTIBLE);
  17758. + schedule();
  17759. + __set_current_state(TASK_RUNNING);
  17760. + continue;
  17761. + }
  17762. +
  17763. + /* Process task list */
  17764. + while (1) {
  17765. + /* save next */
  17766. + next = tsk->posix_timer_list;
  17767. +
  17768. + /* run the task timers, clear its ptr and
  17769. + * unreference it
  17770. + */
  17771. + __run_posix_cpu_timers(tsk);
  17772. + tsk->posix_timer_list = NULL;
  17773. + put_task_struct(tsk);
  17774. +
  17775. + /* check if this is the last on the list */
  17776. + if (next == tsk)
  17777. + break;
  17778. + tsk = next;
  17779. + }
  17780. + }
  17781. + return 0;
  17782. +
  17783. +wait_to_die:
  17784. + /* Wait for kthread_stop */
  17785. + set_current_state(TASK_INTERRUPTIBLE);
  17786. + while (!kthread_should_stop()) {
  17787. + schedule();
  17788. + set_current_state(TASK_INTERRUPTIBLE);
  17789. + }
  17790. + __set_current_state(TASK_RUNNING);
  17791. + return 0;
  17792. +}
  17793. +
  17794. +static inline int __fastpath_timer_check(struct task_struct *tsk)
  17795. +{
  17796. + /* tsk == current, ensure it is safe to use ->signal/sighand */
  17797. + if (unlikely(tsk->exit_state))
  17798. + return 0;
  17799. +
  17800. + if (!task_cputime_zero(&tsk->cputime_expires))
  17801. + return 1;
  17802. +
  17803. + if (!task_cputime_zero(&tsk->signal->cputime_expires))
  17804. + return 1;
  17805. +
  17806. + return 0;
  17807. +}
  17808. +
  17809. +void run_posix_cpu_timers(struct task_struct *tsk)
  17810. +{
  17811. + unsigned long cpu = smp_processor_id();
  17812. + struct task_struct *tasklist;
  17813. +
  17814. + BUG_ON(!irqs_disabled());
  17815. + if(!per_cpu(posix_timer_task, cpu))
  17816. + return;
  17817. + /* get per-cpu references */
  17818. + tasklist = per_cpu(posix_timer_tasklist, cpu);
  17819. +
  17820. + /* check to see if we're already queued */
  17821. + if (!tsk->posix_timer_list && __fastpath_timer_check(tsk)) {
  17822. + get_task_struct(tsk);
  17823. + if (tasklist) {
  17824. + tsk->posix_timer_list = tasklist;
  17825. + } else {
  17826. + /*
  17827. + * The list is terminated by a self-pointing
  17828. + * task_struct
  17829. + */
  17830. + tsk->posix_timer_list = tsk;
  17831. + }
  17832. + per_cpu(posix_timer_tasklist, cpu) = tsk;
  17833. +
  17834. + wake_up_process(per_cpu(posix_timer_task, cpu));
  17835. + }
  17836. +}
  17837. +
  17838. +/*
  17839. + * posix_cpu_thread_call - callback that gets triggered when a CPU is added.
  17840. + * Here we can start up the necessary migration thread for the new CPU.
  17841. + */
  17842. +static int posix_cpu_thread_call(struct notifier_block *nfb,
  17843. + unsigned long action, void *hcpu)
  17844. +{
  17845. + int cpu = (long)hcpu;
  17846. + struct task_struct *p;
  17847. + struct sched_param param;
  17848. +
  17849. + switch (action) {
  17850. + case CPU_UP_PREPARE:
  17851. + p = kthread_create(posix_cpu_timers_thread, hcpu,
  17852. + "posixcputmr/%d",cpu);
  17853. + if (IS_ERR(p))
  17854. + return NOTIFY_BAD;
  17855. + p->flags |= PF_NOFREEZE;
  17856. + kthread_bind(p, cpu);
  17857. + /* Must be high prio to avoid getting starved */
  17858. + param.sched_priority = MAX_RT_PRIO-1;
  17859. + sched_setscheduler(p, SCHED_FIFO, &param);
  17860. + per_cpu(posix_timer_task,cpu) = p;
  17861. + break;
  17862. + case CPU_ONLINE:
  17863. + /* Strictly unneccessary, as first user will wake it. */
  17864. + wake_up_process(per_cpu(posix_timer_task,cpu));
  17865. + break;
  17866. +#ifdef CONFIG_HOTPLUG_CPU
  17867. + case CPU_UP_CANCELED:
  17868. + /* Unbind it from offline cpu so it can run. Fall thru. */
  17869. + kthread_bind(per_cpu(posix_timer_task, cpu),
  17870. + cpumask_any(cpu_online_mask));
  17871. + kthread_stop(per_cpu(posix_timer_task,cpu));
  17872. + per_cpu(posix_timer_task,cpu) = NULL;
  17873. + break;
  17874. + case CPU_DEAD:
  17875. + kthread_stop(per_cpu(posix_timer_task,cpu));
  17876. + per_cpu(posix_timer_task,cpu) = NULL;
  17877. + break;
  17878. +#endif
  17879. + }
  17880. + return NOTIFY_OK;
  17881. +}
  17882. +
  17883. +/* Register at highest priority so that task migration (migrate_all_tasks)
  17884. + * happens before everything else.
  17885. + */
  17886. +static struct notifier_block posix_cpu_thread_notifier = {
  17887. + .notifier_call = posix_cpu_thread_call,
  17888. + .priority = 10
  17889. +};
  17890. +
  17891. +static int __init posix_cpu_thread_init(void)
  17892. +{
  17893. + void *hcpu = (void *)(long)smp_processor_id();
  17894. + /* Start one for boot CPU. */
  17895. + unsigned long cpu;
  17896. +
  17897. + /* init the per-cpu posix_timer_tasklets */
  17898. + for_each_possible_cpu(cpu)
  17899. + per_cpu(posix_timer_tasklist, cpu) = NULL;
  17900. +
  17901. + posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_UP_PREPARE, hcpu);
  17902. + posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_ONLINE, hcpu);
  17903. + register_cpu_notifier(&posix_cpu_thread_notifier);
  17904. + return 0;
  17905. +}
  17906. +early_initcall(posix_cpu_thread_init);
  17907. +#else /* CONFIG_PREEMPT_RT_BASE */
  17908. +void run_posix_cpu_timers(struct task_struct *tsk)
  17909. +{
  17910. + __run_posix_cpu_timers(tsk);
  17911. +}
  17912. +#endif /* CONFIG_PREEMPT_RT_BASE */
  17913. +
  17914. /*
  17915. * Set one of the process-wide special case CPU timers or RLIMIT_CPU.
  17916. * The tsk->sighand->siglock must be held by the caller.
  17917. diff -Nur linux-4.6.6.orig/kernel/time/posix-timers.c linux-4.6.6/kernel/time/posix-timers.c
  17918. --- linux-4.6.6.orig/kernel/time/posix-timers.c 2016-08-10 12:54:57.000000000 +0200
  17919. +++ linux-4.6.6/kernel/time/posix-timers.c 2016-08-22 21:27:47.421893628 +0200
  17920. @@ -506,6 +506,7 @@
  17921. static struct pid *good_sigevent(sigevent_t * event)
  17922. {
  17923. struct task_struct *rtn = current->group_leader;
  17924. + int sig = event->sigev_signo;
  17925. if ((event->sigev_notify & SIGEV_THREAD_ID ) &&
  17926. (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) ||
  17927. @@ -514,7 +515,8 @@
  17928. return NULL;
  17929. if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) &&
  17930. - ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX)))
  17931. + (sig <= 0 || sig > SIGRTMAX || sig_kernel_only(sig) ||
  17932. + sig_kernel_coredump(sig)))
  17933. return NULL;
  17934. return task_pid(rtn);
  17935. @@ -826,6 +828,20 @@
  17936. return overrun;
  17937. }
  17938. +/*
  17939. + * Protected by RCU!
  17940. + */
  17941. +static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr)
  17942. +{
  17943. +#ifdef CONFIG_PREEMPT_RT_FULL
  17944. + if (kc->timer_set == common_timer_set)
  17945. + hrtimer_wait_for_timer(&timr->it.real.timer);
  17946. + else
  17947. + /* FIXME: Whacky hack for posix-cpu-timers */
  17948. + schedule_timeout(1);
  17949. +#endif
  17950. +}
  17951. +
  17952. /* Set a POSIX.1b interval timer. */
  17953. /* timr->it_lock is taken. */
  17954. static int
  17955. @@ -903,6 +919,7 @@
  17956. if (!timr)
  17957. return -EINVAL;
  17958. + rcu_read_lock();
  17959. kc = clockid_to_kclock(timr->it_clock);
  17960. if (WARN_ON_ONCE(!kc || !kc->timer_set))
  17961. error = -EINVAL;
  17962. @@ -911,9 +928,12 @@
  17963. unlock_timer(timr, flag);
  17964. if (error == TIMER_RETRY) {
  17965. + timer_wait_for_callback(kc, timr);
  17966. rtn = NULL; // We already got the old time...
  17967. + rcu_read_unlock();
  17968. goto retry;
  17969. }
  17970. + rcu_read_unlock();
  17971. if (old_setting && !error &&
  17972. copy_to_user(old_setting, &old_spec, sizeof (old_spec)))
  17973. @@ -951,10 +971,15 @@
  17974. if (!timer)
  17975. return -EINVAL;
  17976. + rcu_read_lock();
  17977. if (timer_delete_hook(timer) == TIMER_RETRY) {
  17978. unlock_timer(timer, flags);
  17979. + timer_wait_for_callback(clockid_to_kclock(timer->it_clock),
  17980. + timer);
  17981. + rcu_read_unlock();
  17982. goto retry_delete;
  17983. }
  17984. + rcu_read_unlock();
  17985. spin_lock(&current->sighand->siglock);
  17986. list_del(&timer->list);
  17987. @@ -980,8 +1005,18 @@
  17988. retry_delete:
  17989. spin_lock_irqsave(&timer->it_lock, flags);
  17990. + /* On RT we can race with a deletion */
  17991. + if (!timer->it_signal) {
  17992. + unlock_timer(timer, flags);
  17993. + return;
  17994. + }
  17995. +
  17996. if (timer_delete_hook(timer) == TIMER_RETRY) {
  17997. + rcu_read_lock();
  17998. unlock_timer(timer, flags);
  17999. + timer_wait_for_callback(clockid_to_kclock(timer->it_clock),
  18000. + timer);
  18001. + rcu_read_unlock();
  18002. goto retry_delete;
  18003. }
  18004. list_del(&timer->list);
  18005. diff -Nur linux-4.6.6.orig/kernel/time/tick-broadcast-hrtimer.c linux-4.6.6/kernel/time/tick-broadcast-hrtimer.c
  18006. --- linux-4.6.6.orig/kernel/time/tick-broadcast-hrtimer.c 2016-08-10 12:54:57.000000000 +0200
  18007. +++ linux-4.6.6/kernel/time/tick-broadcast-hrtimer.c 2016-08-22 21:27:47.421893628 +0200
  18008. @@ -106,5 +106,6 @@
  18009. {
  18010. hrtimer_init(&bctimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  18011. bctimer.function = bc_handler;
  18012. + bctimer.irqsafe = true;
  18013. clockevents_register_device(&ce_broadcast_hrtimer);
  18014. }
  18015. diff -Nur linux-4.6.6.orig/kernel/time/tick-common.c linux-4.6.6/kernel/time/tick-common.c
  18016. --- linux-4.6.6.orig/kernel/time/tick-common.c 2016-08-10 12:54:57.000000000 +0200
  18017. +++ linux-4.6.6/kernel/time/tick-common.c 2016-08-22 21:27:47.421893628 +0200
  18018. @@ -79,13 +79,15 @@
  18019. static void tick_periodic(int cpu)
  18020. {
  18021. if (tick_do_timer_cpu == cpu) {
  18022. - write_seqlock(&jiffies_lock);
  18023. + raw_spin_lock(&jiffies_lock);
  18024. + write_seqcount_begin(&jiffies_seq);
  18025. /* Keep track of the next tick event */
  18026. tick_next_period = ktime_add(tick_next_period, tick_period);
  18027. do_timer(1);
  18028. - write_sequnlock(&jiffies_lock);
  18029. + write_seqcount_end(&jiffies_seq);
  18030. + raw_spin_unlock(&jiffies_lock);
  18031. update_wall_time();
  18032. }
  18033. @@ -157,9 +159,9 @@
  18034. ktime_t next;
  18035. do {
  18036. - seq = read_seqbegin(&jiffies_lock);
  18037. + seq = read_seqcount_begin(&jiffies_seq);
  18038. next = tick_next_period;
  18039. - } while (read_seqretry(&jiffies_lock, seq));
  18040. + } while (read_seqcount_retry(&jiffies_seq, seq));
  18041. clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
  18042. diff -Nur linux-4.6.6.orig/kernel/time/tick-internal.h linux-4.6.6/kernel/time/tick-internal.h
  18043. --- linux-4.6.6.orig/kernel/time/tick-internal.h 2016-08-10 12:54:57.000000000 +0200
  18044. +++ linux-4.6.6/kernel/time/tick-internal.h 2016-08-22 21:27:47.421893628 +0200
  18045. @@ -164,3 +164,4 @@
  18046. DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases);
  18047. extern u64 get_next_timer_interrupt(unsigned long basej, u64 basem);
  18048. +void timer_clear_idle(void);
  18049. diff -Nur linux-4.6.6.orig/kernel/time/tick-sched.c linux-4.6.6/kernel/time/tick-sched.c
  18050. --- linux-4.6.6.orig/kernel/time/tick-sched.c 2016-08-10 12:54:57.000000000 +0200
  18051. +++ linux-4.6.6/kernel/time/tick-sched.c 2016-08-22 21:27:47.421893628 +0200
  18052. @@ -62,7 +62,8 @@
  18053. return;
  18054. /* Reevalute with jiffies_lock held */
  18055. - write_seqlock(&jiffies_lock);
  18056. + raw_spin_lock(&jiffies_lock);
  18057. + write_seqcount_begin(&jiffies_seq);
  18058. delta = ktime_sub(now, last_jiffies_update);
  18059. if (delta.tv64 >= tick_period.tv64) {
  18060. @@ -85,10 +86,12 @@
  18061. /* Keep the tick_next_period variable up to date */
  18062. tick_next_period = ktime_add(last_jiffies_update, tick_period);
  18063. } else {
  18064. - write_sequnlock(&jiffies_lock);
  18065. + write_seqcount_end(&jiffies_seq);
  18066. + raw_spin_unlock(&jiffies_lock);
  18067. return;
  18068. }
  18069. - write_sequnlock(&jiffies_lock);
  18070. + write_seqcount_end(&jiffies_seq);
  18071. + raw_spin_unlock(&jiffies_lock);
  18072. update_wall_time();
  18073. }
  18074. @@ -99,12 +102,14 @@
  18075. {
  18076. ktime_t period;
  18077. - write_seqlock(&jiffies_lock);
  18078. + raw_spin_lock(&jiffies_lock);
  18079. + write_seqcount_begin(&jiffies_seq);
  18080. /* Did we start the jiffies update yet ? */
  18081. if (last_jiffies_update.tv64 == 0)
  18082. last_jiffies_update = tick_next_period;
  18083. period = last_jiffies_update;
  18084. - write_sequnlock(&jiffies_lock);
  18085. + write_seqcount_end(&jiffies_seq);
  18086. + raw_spin_unlock(&jiffies_lock);
  18087. return period;
  18088. }
  18089. @@ -212,6 +217,7 @@
  18090. static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
  18091. .func = nohz_full_kick_func,
  18092. + .flags = IRQ_WORK_HARD_IRQ,
  18093. };
  18094. /*
  18095. @@ -670,10 +676,10 @@
  18096. /* Read jiffies and the time when jiffies were updated last */
  18097. do {
  18098. - seq = read_seqbegin(&jiffies_lock);
  18099. + seq = read_seqcount_begin(&jiffies_seq);
  18100. basemono = last_jiffies_update.tv64;
  18101. basejiff = jiffies;
  18102. - } while (read_seqretry(&jiffies_lock, seq));
  18103. + } while (read_seqcount_retry(&jiffies_seq, seq));
  18104. ts->last_jiffies = basejiff;
  18105. if (rcu_needs_cpu(basemono, &next_rcu) ||
  18106. @@ -700,6 +706,12 @@
  18107. delta = next_tick - basemono;
  18108. if (delta <= (u64)TICK_NSEC) {
  18109. tick.tv64 = 0;
  18110. +
  18111. + /*
  18112. + * Tell the timer code that the base is not idle, i.e. undo
  18113. + * the effect of get_next_timer_interrupt().
  18114. + */
  18115. + timer_clear_idle();
  18116. /*
  18117. * We've not stopped the tick yet, and there's a timer in the
  18118. * next period, so no point in stopping it either, bail.
  18119. @@ -808,6 +820,12 @@
  18120. tick_do_update_jiffies64(now);
  18121. update_cpu_load_nohz(active);
  18122. + /*
  18123. + * Clear the timer idle flag, so we avoid IPIs on remote queueing and
  18124. + * the clock forward checks in the enqueue path.
  18125. + */
  18126. + timer_clear_idle();
  18127. +
  18128. calc_load_exit_idle();
  18129. touch_softlockup_watchdog_sched();
  18130. /*
  18131. @@ -861,14 +879,7 @@
  18132. return false;
  18133. if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
  18134. - static int ratelimit;
  18135. -
  18136. - if (ratelimit < 10 &&
  18137. - (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
  18138. - pr_warn("NOHZ: local_softirq_pending %02x\n",
  18139. - (unsigned int) local_softirq_pending());
  18140. - ratelimit++;
  18141. - }
  18142. + softirq_check_pending_idle();
  18143. return false;
  18144. }
  18145. @@ -1091,35 +1102,6 @@
  18146. tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
  18147. }
  18148. -/*
  18149. - * When NOHZ is enabled and the tick is stopped, we need to kick the
  18150. - * tick timer from irq_enter() so that the jiffies update is kept
  18151. - * alive during long running softirqs. That's ugly as hell, but
  18152. - * correctness is key even if we need to fix the offending softirq in
  18153. - * the first place.
  18154. - *
  18155. - * Note, this is different to tick_nohz_restart. We just kick the
  18156. - * timer and do not touch the other magic bits which need to be done
  18157. - * when idle is left.
  18158. - */
  18159. -static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)
  18160. -{
  18161. -#if 0
  18162. - /* Switch back to 2.6.27 behaviour */
  18163. - ktime_t delta;
  18164. -
  18165. - /*
  18166. - * Do not touch the tick device, when the next expiry is either
  18167. - * already reached or less/equal than the tick period.
  18168. - */
  18169. - delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
  18170. - if (delta.tv64 <= tick_period.tv64)
  18171. - return;
  18172. -
  18173. - tick_nohz_restart(ts, now);
  18174. -#endif
  18175. -}
  18176. -
  18177. static inline void tick_nohz_irq_enter(void)
  18178. {
  18179. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  18180. @@ -1130,10 +1112,8 @@
  18181. now = ktime_get();
  18182. if (ts->idle_active)
  18183. tick_nohz_stop_idle(ts, now);
  18184. - if (ts->tick_stopped) {
  18185. + if (ts->tick_stopped)
  18186. tick_nohz_update_jiffies(now);
  18187. - tick_nohz_kick_tick(ts, now);
  18188. - }
  18189. }
  18190. #else
  18191. @@ -1208,6 +1188,7 @@
  18192. * Emulate tick processing via per-CPU hrtimers:
  18193. */
  18194. hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  18195. + ts->sched_timer.irqsafe = 1;
  18196. ts->sched_timer.function = tick_sched_timer;
  18197. /* Get the next period (per cpu) */
  18198. diff -Nur linux-4.6.6.orig/kernel/time/timekeeping.c linux-4.6.6/kernel/time/timekeeping.c
  18199. --- linux-4.6.6.orig/kernel/time/timekeeping.c 2016-08-10 12:54:57.000000000 +0200
  18200. +++ linux-4.6.6/kernel/time/timekeeping.c 2016-08-22 21:27:47.421893628 +0200
  18201. @@ -2319,8 +2319,10 @@
  18202. */
  18203. void xtime_update(unsigned long ticks)
  18204. {
  18205. - write_seqlock(&jiffies_lock);
  18206. + raw_spin_lock(&jiffies_lock);
  18207. + write_seqcount_begin(&jiffies_seq);
  18208. do_timer(ticks);
  18209. - write_sequnlock(&jiffies_lock);
  18210. + write_seqcount_end(&jiffies_seq);
  18211. + raw_spin_unlock(&jiffies_lock);
  18212. update_wall_time();
  18213. }
  18214. diff -Nur linux-4.6.6.orig/kernel/time/timekeeping.h linux-4.6.6/kernel/time/timekeeping.h
  18215. --- linux-4.6.6.orig/kernel/time/timekeeping.h 2016-08-10 12:54:57.000000000 +0200
  18216. +++ linux-4.6.6/kernel/time/timekeeping.h 2016-08-22 21:27:47.425893881 +0200
  18217. @@ -19,7 +19,8 @@
  18218. extern void do_timer(unsigned long ticks);
  18219. extern void update_wall_time(void);
  18220. -extern seqlock_t jiffies_lock;
  18221. +extern raw_spinlock_t jiffies_lock;
  18222. +extern seqcount_t jiffies_seq;
  18223. #define CS_NAME_LEN 32
  18224. diff -Nur linux-4.6.6.orig/kernel/time/timer.c linux-4.6.6/kernel/time/timer.c
  18225. --- linux-4.6.6.orig/kernel/time/timer.c 2016-08-10 12:54:57.000000000 +0200
  18226. +++ linux-4.6.6/kernel/time/timer.c 2016-08-22 21:27:47.425893881 +0200
  18227. @@ -59,43 +59,156 @@
  18228. EXPORT_SYMBOL(jiffies_64);
  18229. /*
  18230. - * per-CPU timer vector definitions:
  18231. + * The timer wheel has LVL_DEPTH array levels. Each level provides an array of
  18232. + * LVL_SIZE buckets. Each level is driven by its own clock and therefor each
  18233. + * level has a different granularity.
  18234. + *
  18235. + * The level granularity is: LVL_CLK_DIV ^ lvl
  18236. + * The level clock frequency is: HZ / (LVL_CLK_DIV ^ level)
  18237. + *
  18238. + * The array level of a newly armed timer depends on the relative expiry
  18239. + * time. The farther the expiry time is away the higher the array level and
  18240. + * therefor the granularity becomes.
  18241. + *
  18242. + * Contrary to the original timer wheel implementation, which aims for 'exact'
  18243. + * expiry of the timers, this implementation removes the need for recascading
  18244. + * the timers into the lower array levels. The previous 'classic' timer wheel
  18245. + * implementation of the kernel already violated the 'exact' expiry by adding
  18246. + * slack to the expiry time to provide batched expiration. The granularity
  18247. + * levels provide implicit batching.
  18248. + *
  18249. + * This is an optimization of the original timer wheel implementation for the
  18250. + * majority of the timer wheel use cases: timeouts. The vast majority of
  18251. + * timeout timers (networking, disk I/O ...) are canceled before expiry. If
  18252. + * the timeout expires it indicates that normal operation is disturbed, so it
  18253. + * does not matter much whether the timeout comes with a slight delay.
  18254. + *
  18255. + * The only exception to this are networking timers with a small expiry
  18256. + * time. They rely on the granularity. Those fit into the first wheel level,
  18257. + * which has HZ granularity.
  18258. + *
  18259. + * We don't have cascading anymore. timers with a expiry time above the
  18260. + * capacity of the last wheel level are force expired at the maximum timeout
  18261. + * value of the last wheel level. From data sampling we know that the maximum
  18262. + * value observed is 5 days (network connection tracking), so this should not
  18263. + * be an issue.
  18264. + *
  18265. + * The currently chosen array constants values are a good compromise between
  18266. + * array size and granularity.
  18267. + *
  18268. + * This results in the following granularity and range levels:
  18269. + *
  18270. + * HZ 1000 steps
  18271. + * Level Offset Granularity Range
  18272. + * 0 0 1 ms 0 ms - 63 ms
  18273. + * 1 64 8 ms 64 ms - 511 ms
  18274. + * 2 128 64 ms 512 ms - 4095 ms (512ms - ~4s)
  18275. + * 3 192 512 ms 4096 ms - 32767 ms (~4s - ~32s)
  18276. + * 4 256 4096 ms (~4s) 32768 ms - 262143 ms (~32s - ~4m)
  18277. + * 5 320 32768 ms (~32s) 262144 ms - 2097151 ms (~4m - ~34m)
  18278. + * 6 384 262144 ms (~4m) 2097152 ms - 16777215 ms (~34m - ~4h)
  18279. + * 7 448 2097152 ms (~34m) 16777216 ms - 134217727 ms (~4h - ~1d)
  18280. + * 8 512 16777216 ms (~4h) 134217728 ms - 1073741822 ms (~1d - ~12d)
  18281. + *
  18282. + * HZ 300
  18283. + * Level Offset Granularity Range
  18284. + * 0 0 3 ms 0 ms - 210 ms
  18285. + * 1 64 26 ms 213 ms - 1703 ms (213ms - ~1s)
  18286. + * 2 128 213 ms 1706 ms - 13650 ms (~1s - ~13s)
  18287. + * 3 192 1706 ms (~1s) 13653 ms - 109223 ms (~13s - ~1m)
  18288. + * 4 256 13653 ms (~13s) 109226 ms - 873810 ms (~1m - ~14m)
  18289. + * 5 320 109226 ms (~1m) 873813 ms - 6990503 ms (~14m - ~1h)
  18290. + * 6 384 873813 ms (~14m) 6990506 ms - 55924050 ms (~1h - ~15h)
  18291. + * 7 448 6990506 ms (~1h) 55924053 ms - 447392423 ms (~15h - ~5d)
  18292. + * 8 512 55924053 ms (~15h) 447392426 ms - 3579139406 ms (~5d - ~41d)
  18293. + *
  18294. + * HZ 250
  18295. + * Level Offset Granularity Range
  18296. + * 0 0 4 ms 0 ms - 255 ms
  18297. + * 1 64 32 ms 256 ms - 2047 ms (256ms - ~2s)
  18298. + * 2 128 256 ms 2048 ms - 16383 ms (~2s - ~16s)
  18299. + * 3 192 2048 ms (~2s) 16384 ms - 131071 ms (~16s - ~2m)
  18300. + * 4 256 16384 ms (~16s) 131072 ms - 1048575 ms (~2m - ~17m)
  18301. + * 5 320 131072 ms (~2m) 1048576 ms - 8388607 ms (~17m - ~2h)
  18302. + * 6 384 1048576 ms (~17m) 8388608 ms - 67108863 ms (~2h - ~18h)
  18303. + * 7 448 8388608 ms (~2h) 67108864 ms - 536870911 ms (~18h - ~6d)
  18304. + * 8 512 67108864 ms (~18h) 536870912 ms - 4294967288 ms (~6d - ~49d)
  18305. + *
  18306. + * HZ 100
  18307. + * Level Offset Granularity Range
  18308. + * 0 0 10 ms 0 ms - 630 ms
  18309. + * 1 64 80 ms 640 ms - 5110 ms (640ms - ~5s)
  18310. + * 2 128 640 ms 5120 ms - 40950 ms (~5s - ~40s)
  18311. + * 3 192 5120 ms (~5s) 40960 ms - 327670 ms (~40s - ~5m)
  18312. + * 4 256 40960 ms (~40s) 327680 ms - 2621430 ms (~5m - ~43m)
  18313. + * 5 320 327680 ms (~5m) 2621440 ms - 20971510 ms (~43m - ~5h)
  18314. + * 6 384 2621440 ms (~43m) 20971520 ms - 167772150 ms (~5h - ~1d)
  18315. + * 7 448 20971520 ms (~5h) 167772160 ms - 1342177270 ms (~1d - ~15d)
  18316. + */
  18317. +
  18318. +/* Clock divisor for the next level */
  18319. +#define LVL_CLK_SHIFT 3
  18320. +#define LVL_CLK_DIV (1UL << LVL_CLK_SHIFT)
  18321. +#define LVL_CLK_MASK (LVL_CLK_DIV - 1)
  18322. +#define LVL_SHIFT(n) ((n) * LVL_CLK_SHIFT)
  18323. +#define LVL_GRAN(n) (1UL << LVL_SHIFT(n))
  18324. +
  18325. +/*
  18326. + * The time start value for each level to select the bucket at enqueue
  18327. + * time.
  18328. */
  18329. -#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
  18330. -#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
  18331. -#define TVN_SIZE (1 << TVN_BITS)
  18332. -#define TVR_SIZE (1 << TVR_BITS)
  18333. -#define TVN_MASK (TVN_SIZE - 1)
  18334. -#define TVR_MASK (TVR_SIZE - 1)
  18335. -#define MAX_TVAL ((unsigned long)((1ULL << (TVR_BITS + 4*TVN_BITS)) - 1))
  18336. +#define LVL_START(n) ((LVL_SIZE - 1) << (((n) - 1) * LVL_CLK_SHIFT))
  18337. -struct tvec {
  18338. - struct hlist_head vec[TVN_SIZE];
  18339. -};
  18340. +/* Size of each clock level */
  18341. +#define LVL_BITS 6
  18342. +#define LVL_SIZE (1UL << LVL_BITS)
  18343. +#define LVL_MASK (LVL_SIZE - 1)
  18344. +#define LVL_OFFS(n) ((n) * LVL_SIZE)
  18345. +
  18346. +/* Level depth */
  18347. +#if HZ > 100
  18348. +# define LVL_DEPTH 9
  18349. +# else
  18350. +# define LVL_DEPTH 8
  18351. +#endif
  18352. -struct tvec_root {
  18353. - struct hlist_head vec[TVR_SIZE];
  18354. -};
  18355. +/* The cutoff (max. capacity of the wheel) */
  18356. +#define WHEEL_TIMEOUT_CUTOFF (LVL_START(LVL_DEPTH))
  18357. +#define WHEEL_TIMEOUT_MAX (WHEEL_TIMEOUT_CUTOFF - LVL_GRAN(LVL_DEPTH - 1))
  18358. -struct tvec_base {
  18359. - spinlock_t lock;
  18360. - struct timer_list *running_timer;
  18361. - unsigned long timer_jiffies;
  18362. - unsigned long next_timer;
  18363. - unsigned long active_timers;
  18364. - unsigned long all_timers;
  18365. - int cpu;
  18366. - bool migration_enabled;
  18367. - bool nohz_active;
  18368. - struct tvec_root tv1;
  18369. - struct tvec tv2;
  18370. - struct tvec tv3;
  18371. - struct tvec tv4;
  18372. - struct tvec tv5;
  18373. -} ____cacheline_aligned;
  18374. +/*
  18375. + * The resulting wheel size. If NOHZ is configured we allocate two
  18376. + * wheels so we have a separate storage for the deferrable timers.
  18377. + */
  18378. +#define WHEEL_SIZE (LVL_SIZE * LVL_DEPTH)
  18379. +#ifdef CONFIG_NO_HZ_COMMON
  18380. +# define NR_BASES 2
  18381. +# define BASE_STD 0
  18382. +# define BASE_DEF 1
  18383. +#else
  18384. +# define NR_BASES 1
  18385. +# define BASE_STD 0
  18386. +# define BASE_DEF 0
  18387. +#endif
  18388. -static DEFINE_PER_CPU(struct tvec_base, tvec_bases);
  18389. +struct timer_base {
  18390. + raw_spinlock_t lock;
  18391. + struct timer_list *running_timer;
  18392. +#ifdef CONFIG_PREEMPT_RT_FULL
  18393. + struct swait_queue_head wait_for_running_timer;
  18394. +#endif
  18395. + unsigned long clk;
  18396. + unsigned long next_expiry;
  18397. + unsigned int cpu;
  18398. + bool migration_enabled;
  18399. + bool nohz_active;
  18400. + bool is_idle;
  18401. + DECLARE_BITMAP(pending_map, WHEEL_SIZE);
  18402. + struct hlist_head vectors[WHEEL_SIZE];
  18403. +} ____cacheline_aligned;
  18404. +
  18405. +static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
  18406. #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
  18407. unsigned int sysctl_timer_migration = 1;
  18408. @@ -106,15 +219,17 @@
  18409. unsigned int cpu;
  18410. /* Avoid the loop, if nothing to update */
  18411. - if (this_cpu_read(tvec_bases.migration_enabled) == on)
  18412. + if (this_cpu_read(timer_bases[BASE_STD].migration_enabled) == on)
  18413. return;
  18414. for_each_possible_cpu(cpu) {
  18415. - per_cpu(tvec_bases.migration_enabled, cpu) = on;
  18416. + per_cpu(timer_bases[BASE_STD].migration_enabled, cpu) = on;
  18417. + per_cpu(timer_bases[BASE_DEF].migration_enabled, cpu) = on;
  18418. per_cpu(hrtimer_bases.migration_enabled, cpu) = on;
  18419. if (!update_nohz)
  18420. continue;
  18421. - per_cpu(tvec_bases.nohz_active, cpu) = true;
  18422. + per_cpu(timer_bases[BASE_STD].nohz_active, cpu) = true;
  18423. + per_cpu(timer_bases[BASE_DEF].nohz_active, cpu) = true;
  18424. per_cpu(hrtimer_bases.nohz_active, cpu) = true;
  18425. }
  18426. }
  18427. @@ -133,20 +248,6 @@
  18428. mutex_unlock(&mutex);
  18429. return ret;
  18430. }
  18431. -
  18432. -static inline struct tvec_base *get_target_base(struct tvec_base *base,
  18433. - int pinned)
  18434. -{
  18435. - if (pinned || !base->migration_enabled)
  18436. - return this_cpu_ptr(&tvec_bases);
  18437. - return per_cpu_ptr(&tvec_bases, get_nohz_timer_target());
  18438. -}
  18439. -#else
  18440. -static inline struct tvec_base *get_target_base(struct tvec_base *base,
  18441. - int pinned)
  18442. -{
  18443. - return this_cpu_ptr(&tvec_bases);
  18444. -}
  18445. #endif
  18446. static unsigned long round_jiffies_common(unsigned long j, int cpu,
  18447. @@ -351,101 +452,126 @@
  18448. }
  18449. EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
  18450. -/**
  18451. - * set_timer_slack - set the allowed slack for a timer
  18452. - * @timer: the timer to be modified
  18453. - * @slack_hz: the amount of time (in jiffies) allowed for rounding
  18454. - *
  18455. - * Set the amount of time, in jiffies, that a certain timer has
  18456. - * in terms of slack. By setting this value, the timer subsystem
  18457. - * will schedule the actual timer somewhere between
  18458. - * the time mod_timer() asks for, and that time plus the slack.
  18459. - *
  18460. - * By setting the slack to -1, a percentage of the delay is used
  18461. - * instead.
  18462. - */
  18463. -void set_timer_slack(struct timer_list *timer, int slack_hz)
  18464. +
  18465. +static inline unsigned int timer_get_idx(struct timer_list *timer)
  18466. {
  18467. - timer->slack = slack_hz;
  18468. + return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;
  18469. }
  18470. -EXPORT_SYMBOL_GPL(set_timer_slack);
  18471. -static void
  18472. -__internal_add_timer(struct tvec_base *base, struct timer_list *timer)
  18473. +static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)
  18474. {
  18475. - unsigned long expires = timer->expires;
  18476. - unsigned long idx = expires - base->timer_jiffies;
  18477. - struct hlist_head *vec;
  18478. + timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |
  18479. + idx << TIMER_ARRAYSHIFT;
  18480. +}
  18481. - if (idx < TVR_SIZE) {
  18482. - int i = expires & TVR_MASK;
  18483. - vec = base->tv1.vec + i;
  18484. - } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
  18485. - int i = (expires >> TVR_BITS) & TVN_MASK;
  18486. - vec = base->tv2.vec + i;
  18487. - } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {
  18488. - int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
  18489. - vec = base->tv3.vec + i;
  18490. - } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {
  18491. - int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;
  18492. - vec = base->tv4.vec + i;
  18493. - } else if ((signed long) idx < 0) {
  18494. - /*
  18495. - * Can happen if you add a timer with expires == jiffies,
  18496. - * or you set a timer to go off in the past
  18497. - */
  18498. - vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
  18499. +/*
  18500. + * Helper function to calculate the array index for a given expiry
  18501. + * time.
  18502. + */
  18503. +static inline unsigned calc_index(unsigned expires, unsigned lvl)
  18504. +{
  18505. + expires = (expires + LVL_GRAN(lvl)) >> LVL_SHIFT(lvl);
  18506. + return LVL_OFFS(lvl) + (expires & LVL_MASK);
  18507. +}
  18508. +
  18509. +static int calc_wheel_index(unsigned long expires, unsigned long clk)
  18510. +{
  18511. + unsigned long delta = expires - clk;
  18512. + unsigned int idx;
  18513. +
  18514. + if (delta < LVL_START(1)) {
  18515. + idx = calc_index(expires, 0);
  18516. + } else if (delta < LVL_START(2)) {
  18517. + idx = calc_index(expires, 1);
  18518. + } else if (delta < LVL_START(3)) {
  18519. + idx = calc_index(expires, 2);
  18520. + } else if (delta < LVL_START(4)) {
  18521. + idx = calc_index(expires, 3);
  18522. + } else if (delta < LVL_START(5)) {
  18523. + idx = calc_index(expires, 4);
  18524. + } else if (delta < LVL_START(6)) {
  18525. + idx = calc_index(expires, 5);
  18526. + } else if (delta < LVL_START(7)) {
  18527. + idx = calc_index(expires, 6);
  18528. + } else if (LVL_DEPTH > 8 && delta < LVL_START(8)) {
  18529. + idx = calc_index(expires, 7);
  18530. + } else if ((long) delta < 0) {
  18531. + idx = clk & LVL_MASK;
  18532. } else {
  18533. - int i;
  18534. - /* If the timeout is larger than MAX_TVAL (on 64-bit
  18535. - * architectures or with CONFIG_BASE_SMALL=1) then we
  18536. - * use the maximum timeout.
  18537. + /*
  18538. + * Force expire obscene large timeouts to expire at the
  18539. + * capacity limit of the wheel.
  18540. */
  18541. - if (idx > MAX_TVAL) {
  18542. - idx = MAX_TVAL;
  18543. - expires = idx + base->timer_jiffies;
  18544. - }
  18545. - i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;
  18546. - vec = base->tv5.vec + i;
  18547. + if (expires >= WHEEL_TIMEOUT_CUTOFF)
  18548. + expires = WHEEL_TIMEOUT_MAX;
  18549. +
  18550. + idx = calc_index(expires, LVL_DEPTH - 1);
  18551. }
  18552. + return idx;
  18553. +}
  18554. - hlist_add_head(&timer->entry, vec);
  18555. +/*
  18556. + * Enqueue the timer into the hash bucket, mark it pending in
  18557. + * the bitmap and store the index in the timer flags.
  18558. + */
  18559. +static void enqueue_timer(struct timer_base *base, struct timer_list *timer,
  18560. + unsigned int idx)
  18561. +{
  18562. + hlist_add_head(&timer->entry, base->vectors + idx);
  18563. + __set_bit(idx, base->pending_map);
  18564. + timer_set_idx(timer, idx);
  18565. }
  18566. -static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
  18567. +static void
  18568. +__internal_add_timer(struct timer_base *base, struct timer_list *timer)
  18569. {
  18570. - /* Advance base->jiffies, if the base is empty */
  18571. - if (!base->all_timers++)
  18572. - base->timer_jiffies = jiffies;
  18573. + unsigned int idx;
  18574. +
  18575. + idx = calc_wheel_index(timer->expires, base->clk);
  18576. + enqueue_timer(base, timer, idx);
  18577. +}
  18578. +
  18579. +static void
  18580. +trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
  18581. +{
  18582. + if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active)
  18583. + return;
  18584. - __internal_add_timer(base, timer);
  18585. /*
  18586. - * Update base->active_timers and base->next_timer
  18587. + * This wants some optimizing similar to the below, but we do that
  18588. + * when we switch from push to pull for deferrable timers.
  18589. */
  18590. - if (!(timer->flags & TIMER_DEFERRABLE)) {
  18591. - if (!base->active_timers++ ||
  18592. - time_before(timer->expires, base->next_timer))
  18593. - base->next_timer = timer->expires;
  18594. + if (timer->flags & TIMER_DEFERRABLE) {
  18595. + if (tick_nohz_full_cpu(base->cpu))
  18596. + wake_up_nohz_cpu(base->cpu);
  18597. + return;
  18598. }
  18599. /*
  18600. - * Check whether the other CPU is in dynticks mode and needs
  18601. - * to be triggered to reevaluate the timer wheel.
  18602. - * We are protected against the other CPU fiddling
  18603. - * with the timer by holding the timer base lock. This also
  18604. - * makes sure that a CPU on the way to stop its tick can not
  18605. - * evaluate the timer wheel.
  18606. - *
  18607. - * Spare the IPI for deferrable timers on idle targets though.
  18608. - * The next busy ticks will take care of it. Except full dynticks
  18609. - * require special care against races with idle_cpu(), lets deal
  18610. - * with that later.
  18611. - */
  18612. - if (base->nohz_active) {
  18613. - if (!(timer->flags & TIMER_DEFERRABLE) ||
  18614. - tick_nohz_full_cpu(base->cpu))
  18615. - wake_up_nohz_cpu(base->cpu);
  18616. - }
  18617. + * We might have to IPI the remote CPU if the base is idle and the
  18618. + * timer is not deferrable. If the other cpu is on the way to idle
  18619. + * then it can't set base->is_idle as we hold base lock.
  18620. + */
  18621. + if (!base->is_idle)
  18622. + return;
  18623. +
  18624. + /* Check whether this is the new first expiring timer */
  18625. + if (time_after_eq(timer->expires, base->next_expiry))
  18626. + return;
  18627. +
  18628. + /*
  18629. + * Set the next expiry time and kick the cpu so it can reevaluate the
  18630. + * wheel
  18631. + */
  18632. + base->next_expiry = timer->expires;
  18633. + wake_up_nohz_cpu(base->cpu);
  18634. +}
  18635. +
  18636. +static void
  18637. +internal_add_timer(struct timer_base *base, struct timer_list *timer)
  18638. +{
  18639. + __internal_add_timer(base, timer);
  18640. + trigger_dyntick_cpu(base, timer);
  18641. }
  18642. #ifdef CONFIG_TIMER_STATS
  18643. @@ -681,7 +807,6 @@
  18644. {
  18645. timer->entry.pprev = NULL;
  18646. timer->flags = flags | raw_smp_processor_id();
  18647. - timer->slack = -1;
  18648. #ifdef CONFIG_TIMER_STATS
  18649. timer->start_site = NULL;
  18650. timer->start_pid = -1;
  18651. @@ -721,71 +846,170 @@
  18652. entry->next = LIST_POISON2;
  18653. }
  18654. -static inline void
  18655. -detach_expired_timer(struct timer_list *timer, struct tvec_base *base)
  18656. -{
  18657. - detach_timer(timer, true);
  18658. - if (!(timer->flags & TIMER_DEFERRABLE))
  18659. - base->active_timers--;
  18660. - base->all_timers--;
  18661. -}
  18662. -
  18663. -static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,
  18664. +static int detach_if_pending(struct timer_list *timer, struct timer_base *base,
  18665. bool clear_pending)
  18666. {
  18667. + unsigned idx = timer_get_idx(timer);
  18668. +
  18669. if (!timer_pending(timer))
  18670. return 0;
  18671. + if (hlist_is_singular_node(&timer->entry, base->vectors + idx))
  18672. + __clear_bit(idx, base->pending_map);
  18673. +
  18674. detach_timer(timer, clear_pending);
  18675. - if (!(timer->flags & TIMER_DEFERRABLE)) {
  18676. - base->active_timers--;
  18677. - if (timer->expires == base->next_timer)
  18678. - base->next_timer = base->timer_jiffies;
  18679. - }
  18680. - /* If this was the last timer, advance base->jiffies */
  18681. - if (!--base->all_timers)
  18682. - base->timer_jiffies = jiffies;
  18683. return 1;
  18684. }
  18685. +static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu)
  18686. +{
  18687. + struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_STD], cpu);
  18688. +
  18689. + /*
  18690. + * If the timer is deferrable and nohz is active then we need to use
  18691. + * the deferrable base.
  18692. + */
  18693. + if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active &&
  18694. + (tflags & TIMER_DEFERRABLE))
  18695. + base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu);
  18696. + return base;
  18697. +}
  18698. +
  18699. +static inline struct timer_base *get_timer_this_cpu_base(u32 tflags)
  18700. +{
  18701. + struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  18702. +
  18703. + /*
  18704. + * If the timer is deferrable and nohz is active then we need to use
  18705. + * the deferrable base.
  18706. + */
  18707. + if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active &&
  18708. + (tflags & TIMER_DEFERRABLE))
  18709. + base = this_cpu_ptr(&timer_bases[BASE_DEF]);
  18710. + return base;
  18711. +}
  18712. +
  18713. +static inline struct timer_base *get_timer_base(u32 tflags)
  18714. +{
  18715. + return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
  18716. +}
  18717. +
  18718. +#ifdef CONFIG_NO_HZ_COMMON
  18719. +static inline struct timer_base *__get_target_base(struct timer_base *base,
  18720. + unsigned tflags)
  18721. +{
  18722. +#ifdef CONFIG_SMP
  18723. + if ((tflags & TIMER_PINNED) || !base->migration_enabled)
  18724. + return get_timer_this_cpu_base(tflags);
  18725. + return get_timer_cpu_base(tflags, get_nohz_timer_target());
  18726. +#else
  18727. + return get_timer_this_cpu_base(tflags);
  18728. +#endif
  18729. +}
  18730. +
  18731. +static inline void forward_timer_base(struct timer_base *base)
  18732. +{
  18733. + /*
  18734. + * We only forward the base when it's idle and we have a delta between
  18735. + * base clock and jiffies.
  18736. + */
  18737. + if (!base->is_idle || (long) (jiffies - base->clk) < 2)
  18738. + return;
  18739. +
  18740. + /*
  18741. + * If the next expiry value is > jiffies, then we fast forward to
  18742. + * jiffies otherwise we forward to the next expiry value.
  18743. + */
  18744. + if (time_after(base->next_expiry, jiffies))
  18745. + base->clk = jiffies;
  18746. + else
  18747. + base->clk = base->next_expiry;
  18748. +}
  18749. +#else
  18750. +static inline struct timer_base *__get_target_base(struct timer_base *base,
  18751. + unsigned tflags)
  18752. +{
  18753. + return get_timer_this_cpu_base(tflags);
  18754. +}
  18755. +
  18756. +static inline void forward_timer_base(struct timer_base *base) { }
  18757. +#endif
  18758. +
  18759. +static inline struct timer_base *get_target_base(struct timer_base *base,
  18760. + unsigned tflags)
  18761. +{
  18762. + struct timer_base *target = __get_target_base(base, tflags);
  18763. +
  18764. + forward_timer_base(target);
  18765. + return target;
  18766. +}
  18767. +
  18768. /*
  18769. - * We are using hashed locking: holding per_cpu(tvec_bases).lock
  18770. - * means that all timers which are tied to this base via timer->base are
  18771. - * locked, and the base itself is locked too.
  18772. + * We are using hashed locking: Holding per_cpu(timer_bases[x]).lock means
  18773. + * that all timers which are tied to this base are locked, and the base itself
  18774. + * is locked too.
  18775. *
  18776. * So __run_timers/migrate_timers can safely modify all timers which could
  18777. - * be found on ->tvX lists.
  18778. + * be found in the base->vectors array.
  18779. *
  18780. - * When the timer's base is locked and removed from the list, the
  18781. - * TIMER_MIGRATING flag is set, FIXME
  18782. + * When a timer is migrating then the TIMER_MIGRATING flag is set and we need
  18783. + * to wait until the migration is done.
  18784. */
  18785. -static struct tvec_base *lock_timer_base(struct timer_list *timer,
  18786. - unsigned long *flags)
  18787. +static struct timer_base *lock_timer_base(struct timer_list *timer,
  18788. + unsigned long *flags)
  18789. __acquires(timer->base->lock)
  18790. {
  18791. for (;;) {
  18792. + struct timer_base *base;
  18793. u32 tf = timer->flags;
  18794. - struct tvec_base *base;
  18795. if (!(tf & TIMER_MIGRATING)) {
  18796. - base = per_cpu_ptr(&tvec_bases, tf & TIMER_CPUMASK);
  18797. - spin_lock_irqsave(&base->lock, *flags);
  18798. + base = get_timer_base(tf);
  18799. + raw_spin_lock_irqsave(&base->lock, *flags);
  18800. if (timer->flags == tf)
  18801. return base;
  18802. - spin_unlock_irqrestore(&base->lock, *flags);
  18803. + raw_spin_unlock_irqrestore(&base->lock, *flags);
  18804. }
  18805. cpu_relax();
  18806. }
  18807. }
  18808. static inline int
  18809. -__mod_timer(struct timer_list *timer, unsigned long expires,
  18810. - bool pending_only, int pinned)
  18811. +__mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
  18812. {
  18813. - struct tvec_base *base, *new_base;
  18814. - unsigned long flags;
  18815. + struct timer_base *base, *new_base;
  18816. + unsigned int idx = UINT_MAX;
  18817. + unsigned long clk = 0, flags;
  18818. int ret = 0;
  18819. + /*
  18820. + * This is a common optimization triggered by the networking code - if
  18821. + * the timer is re-modified to be the same thing or ends up in the
  18822. + * same array bucket then just return:
  18823. + */
  18824. + if (timer_pending(timer)) {
  18825. + if (timer->expires == expires)
  18826. + return 1;
  18827. + /*
  18828. + * Take the current timer_jiffies of base, but without holding
  18829. + * the lock!
  18830. + */
  18831. + base = get_timer_base(timer->flags);
  18832. + clk = base->clk;
  18833. +
  18834. + idx = calc_wheel_index(expires, clk);
  18835. +
  18836. + /*
  18837. + * Retrieve and compare the array index of the pending
  18838. + * timer. If it matches set the expiry to the new value so a
  18839. + * subsequent call will exit in the expires check above.
  18840. + */
  18841. + if (idx == timer_get_idx(timer)) {
  18842. + timer->expires = expires;
  18843. + return 1;
  18844. + }
  18845. + }
  18846. +
  18847. timer_stats_timer_set_start_info(timer);
  18848. BUG_ON(!timer->function);
  18849. @@ -797,33 +1021,44 @@
  18850. debug_activate(timer, expires);
  18851. - new_base = get_target_base(base, pinned);
  18852. + new_base = get_target_base(base, timer->flags);
  18853. if (base != new_base) {
  18854. /*
  18855. - * We are trying to schedule the timer on the local CPU.
  18856. + * We are trying to schedule the timer on the new base.
  18857. * However we can't change timer's base while it is running,
  18858. * otherwise del_timer_sync() can't detect that the timer's
  18859. - * handler yet has not finished. This also guarantees that
  18860. - * the timer is serialized wrt itself.
  18861. + * handler yet has not finished. This also guarantees that the
  18862. + * timer is serialized wrt itself.
  18863. */
  18864. if (likely(base->running_timer != timer)) {
  18865. /* See the comment in lock_timer_base() */
  18866. timer->flags |= TIMER_MIGRATING;
  18867. - spin_unlock(&base->lock);
  18868. + raw_spin_unlock(&base->lock);
  18869. base = new_base;
  18870. - spin_lock(&base->lock);
  18871. + raw_spin_lock(&base->lock);
  18872. WRITE_ONCE(timer->flags,
  18873. (timer->flags & ~TIMER_BASEMASK) | base->cpu);
  18874. }
  18875. }
  18876. timer->expires = expires;
  18877. - internal_add_timer(base, timer);
  18878. + /*
  18879. + * If idx was calculated above and the base time did not advance
  18880. + * between calculating idx and taking the lock, only enqueue_timer()
  18881. + * and trigger_dyntick_cpu() is required. Otherwise we need to
  18882. + * (re)calculate the wheel index via internal_add_timer().
  18883. + */
  18884. + if (idx != UINT_MAX && clk == base->clk) {
  18885. + enqueue_timer(base, timer, idx);
  18886. + trigger_dyntick_cpu(base, timer);
  18887. + } else {
  18888. + internal_add_timer(base, timer);
  18889. + }
  18890. out_unlock:
  18891. - spin_unlock_irqrestore(&base->lock, flags);
  18892. + raw_spin_unlock_irqrestore(&base->lock, flags);
  18893. return ret;
  18894. }
  18895. @@ -840,49 +1075,10 @@
  18896. */
  18897. int mod_timer_pending(struct timer_list *timer, unsigned long expires)
  18898. {
  18899. - return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);
  18900. + return __mod_timer(timer, expires, true);
  18901. }
  18902. EXPORT_SYMBOL(mod_timer_pending);
  18903. -/*
  18904. - * Decide where to put the timer while taking the slack into account
  18905. - *
  18906. - * Algorithm:
  18907. - * 1) calculate the maximum (absolute) time
  18908. - * 2) calculate the highest bit where the expires and new max are different
  18909. - * 3) use this bit to make a mask
  18910. - * 4) use the bitmask to round down the maximum time, so that all last
  18911. - * bits are zeros
  18912. - */
  18913. -static inline
  18914. -unsigned long apply_slack(struct timer_list *timer, unsigned long expires)
  18915. -{
  18916. - unsigned long expires_limit, mask;
  18917. - int bit;
  18918. -
  18919. - if (timer->slack >= 0) {
  18920. - expires_limit = expires + timer->slack;
  18921. - } else {
  18922. - long delta = expires - jiffies;
  18923. -
  18924. - if (delta < 256)
  18925. - return expires;
  18926. -
  18927. - expires_limit = expires + delta / 256;
  18928. - }
  18929. - mask = expires ^ expires_limit;
  18930. - if (mask == 0)
  18931. - return expires;
  18932. -
  18933. - bit = __fls(mask);
  18934. -
  18935. - mask = (1UL << bit) - 1;
  18936. -
  18937. - expires_limit = expires_limit & ~(mask);
  18938. -
  18939. - return expires_limit;
  18940. -}
  18941. -
  18942. /**
  18943. * mod_timer - modify a timer's timeout
  18944. * @timer: the timer to be modified
  18945. @@ -905,49 +1101,11 @@
  18946. */
  18947. int mod_timer(struct timer_list *timer, unsigned long expires)
  18948. {
  18949. - expires = apply_slack(timer, expires);
  18950. -
  18951. - /*
  18952. - * This is a common optimization triggered by the
  18953. - * networking code - if the timer is re-modified
  18954. - * to be the same thing then just return:
  18955. - */
  18956. - if (timer_pending(timer) && timer->expires == expires)
  18957. - return 1;
  18958. -
  18959. - return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);
  18960. + return __mod_timer(timer, expires, false);
  18961. }
  18962. EXPORT_SYMBOL(mod_timer);
  18963. /**
  18964. - * mod_timer_pinned - modify a timer's timeout
  18965. - * @timer: the timer to be modified
  18966. - * @expires: new timeout in jiffies
  18967. - *
  18968. - * mod_timer_pinned() is a way to update the expire field of an
  18969. - * active timer (if the timer is inactive it will be activated)
  18970. - * and to ensure that the timer is scheduled on the current CPU.
  18971. - *
  18972. - * Note that this does not prevent the timer from being migrated
  18973. - * when the current CPU goes offline. If this is a problem for
  18974. - * you, use CPU-hotplug notifiers to handle it correctly, for
  18975. - * example, cancelling the timer when the corresponding CPU goes
  18976. - * offline.
  18977. - *
  18978. - * mod_timer_pinned(timer, expires) is equivalent to:
  18979. - *
  18980. - * del_timer(timer); timer->expires = expires; add_timer(timer);
  18981. - */
  18982. -int mod_timer_pinned(struct timer_list *timer, unsigned long expires)
  18983. -{
  18984. - if (timer->expires == expires && timer_pending(timer))
  18985. - return 1;
  18986. -
  18987. - return __mod_timer(timer, expires, false, TIMER_PINNED);
  18988. -}
  18989. -EXPORT_SYMBOL(mod_timer_pinned);
  18990. -
  18991. -/**
  18992. * add_timer - start a timer
  18993. * @timer: the timer to be added
  18994. *
  18995. @@ -977,13 +1135,14 @@
  18996. */
  18997. void add_timer_on(struct timer_list *timer, int cpu)
  18998. {
  18999. - struct tvec_base *new_base = per_cpu_ptr(&tvec_bases, cpu);
  19000. - struct tvec_base *base;
  19001. + struct timer_base *new_base, *base;
  19002. unsigned long flags;
  19003. timer_stats_timer_set_start_info(timer);
  19004. BUG_ON(timer_pending(timer) || !timer->function);
  19005. + new_base = get_timer_cpu_base(timer->flags, cpu);
  19006. +
  19007. /*
  19008. * If @timer was on a different CPU, it should be migrated with the
  19009. * old base locked to prevent other operations proceeding with the
  19010. @@ -993,19 +1152,46 @@
  19011. if (base != new_base) {
  19012. timer->flags |= TIMER_MIGRATING;
  19013. - spin_unlock(&base->lock);
  19014. + raw_spin_unlock(&base->lock);
  19015. base = new_base;
  19016. - spin_lock(&base->lock);
  19017. + raw_spin_lock(&base->lock);
  19018. WRITE_ONCE(timer->flags,
  19019. (timer->flags & ~TIMER_BASEMASK) | cpu);
  19020. }
  19021. debug_activate(timer, timer->expires);
  19022. internal_add_timer(base, timer);
  19023. - spin_unlock_irqrestore(&base->lock, flags);
  19024. + raw_spin_unlock_irqrestore(&base->lock, flags);
  19025. }
  19026. EXPORT_SYMBOL_GPL(add_timer_on);
  19027. +#ifdef CONFIG_PREEMPT_RT_FULL
  19028. +/*
  19029. + * Wait for a running timer
  19030. + */
  19031. +static void wait_for_running_timer(struct timer_list *timer)
  19032. +{
  19033. + struct timer_base *base;
  19034. + u32 tf = timer->flags;
  19035. +
  19036. + if (tf & TIMER_MIGRATING)
  19037. + return;
  19038. +
  19039. + base = get_timer_base(tf);
  19040. + swait_event(base->wait_for_running_timer,
  19041. + base->running_timer != timer);
  19042. +}
  19043. +
  19044. +# define wakeup_timer_waiters(b) swake_up_all(&(b)->wait_for_running_timer)
  19045. +#else
  19046. +static inline void wait_for_running_timer(struct timer_list *timer)
  19047. +{
  19048. + cpu_relax();
  19049. +}
  19050. +
  19051. +# define wakeup_timer_waiters(b) do { } while (0)
  19052. +#endif
  19053. +
  19054. /**
  19055. * del_timer - deactive a timer.
  19056. * @timer: the timer to be deactivated
  19057. @@ -1019,7 +1205,7 @@
  19058. */
  19059. int del_timer(struct timer_list *timer)
  19060. {
  19061. - struct tvec_base *base;
  19062. + struct timer_base *base;
  19063. unsigned long flags;
  19064. int ret = 0;
  19065. @@ -1029,7 +1215,7 @@
  19066. if (timer_pending(timer)) {
  19067. base = lock_timer_base(timer, &flags);
  19068. ret = detach_if_pending(timer, base, true);
  19069. - spin_unlock_irqrestore(&base->lock, flags);
  19070. + raw_spin_unlock_irqrestore(&base->lock, flags);
  19071. }
  19072. return ret;
  19073. @@ -1045,7 +1231,7 @@
  19074. */
  19075. int try_to_del_timer_sync(struct timer_list *timer)
  19076. {
  19077. - struct tvec_base *base;
  19078. + struct timer_base *base;
  19079. unsigned long flags;
  19080. int ret = -1;
  19081. @@ -1057,13 +1243,13 @@
  19082. timer_stats_timer_clear_start_info(timer);
  19083. ret = detach_if_pending(timer, base, true);
  19084. }
  19085. - spin_unlock_irqrestore(&base->lock, flags);
  19086. + raw_spin_unlock_irqrestore(&base->lock, flags);
  19087. return ret;
  19088. }
  19089. EXPORT_SYMBOL(try_to_del_timer_sync);
  19090. -#ifdef CONFIG_SMP
  19091. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  19092. /**
  19093. * del_timer_sync - deactivate a timer and wait for the handler to finish.
  19094. * @timer: the timer to be deactivated
  19095. @@ -1123,33 +1309,12 @@
  19096. int ret = try_to_del_timer_sync(timer);
  19097. if (ret >= 0)
  19098. return ret;
  19099. - cpu_relax();
  19100. + wait_for_running_timer(timer);
  19101. }
  19102. }
  19103. EXPORT_SYMBOL(del_timer_sync);
  19104. #endif
  19105. -static int cascade(struct tvec_base *base, struct tvec *tv, int index)
  19106. -{
  19107. - /* cascade all the timers from tv up one level */
  19108. - struct timer_list *timer;
  19109. - struct hlist_node *tmp;
  19110. - struct hlist_head tv_list;
  19111. -
  19112. - hlist_move_list(tv->vec + index, &tv_list);
  19113. -
  19114. - /*
  19115. - * We are removing _all_ timers from the list, so we
  19116. - * don't have to detach them individually.
  19117. - */
  19118. - hlist_for_each_entry_safe(timer, tmp, &tv_list, entry) {
  19119. - /* No accounting, while moving them */
  19120. - __internal_add_timer(base, timer);
  19121. - }
  19122. -
  19123. - return index;
  19124. -}
  19125. -
  19126. static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),
  19127. unsigned long data)
  19128. {
  19129. @@ -1193,147 +1358,144 @@
  19130. }
  19131. }
  19132. -#define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
  19133. -
  19134. -/**
  19135. - * __run_timers - run all expired timers (if any) on this CPU.
  19136. - * @base: the timer vector to be processed.
  19137. - *
  19138. - * This function cascades all vectors and executes all expired timer
  19139. - * vectors.
  19140. - */
  19141. -static inline void __run_timers(struct tvec_base *base)
  19142. +static void expire_timers(struct timer_base *base, struct hlist_head *head)
  19143. {
  19144. - struct timer_list *timer;
  19145. + while (!hlist_empty(head)) {
  19146. + struct timer_list *timer;
  19147. + void (*fn)(unsigned long);
  19148. + unsigned long data;
  19149. +
  19150. + timer = hlist_entry(head->first, struct timer_list, entry);
  19151. + timer_stats_account_timer(timer);
  19152. - spin_lock_irq(&base->lock);
  19153. + base->running_timer = timer;
  19154. + detach_timer(timer, true);
  19155. - while (time_after_eq(jiffies, base->timer_jiffies)) {
  19156. - struct hlist_head work_list;
  19157. - struct hlist_head *head = &work_list;
  19158. - int index;
  19159. + fn = timer->function;
  19160. + data = timer->data;
  19161. - if (!base->all_timers) {
  19162. - base->timer_jiffies = jiffies;
  19163. - break;
  19164. + if (!IS_ENABLED(CONFIG_PREEMPT_RT_FULL) &&
  19165. + timer->flags & TIMER_IRQSAFE) {
  19166. + raw_spin_unlock(&base->lock);
  19167. + call_timer_fn(timer, fn, data);
  19168. + base->running_timer = NULL;
  19169. + raw_spin_lock(&base->lock);
  19170. + } else {
  19171. + raw_spin_unlock_irq(&base->lock);
  19172. + call_timer_fn(timer, fn, data);
  19173. + base->running_timer = NULL;
  19174. + raw_spin_lock_irq(&base->lock);
  19175. }
  19176. + }
  19177. +}
  19178. - index = base->timer_jiffies & TVR_MASK;
  19179. +static int __collect_expired_timers(struct timer_base *base,
  19180. + struct hlist_head *heads)
  19181. +{
  19182. + unsigned long clk = base->clk;
  19183. + struct hlist_head *vec;
  19184. + int i, levels = 0;
  19185. + unsigned int idx;
  19186. - /*
  19187. - * Cascade timers:
  19188. - */
  19189. - if (!index &&
  19190. - (!cascade(base, &base->tv2, INDEX(0))) &&
  19191. - (!cascade(base, &base->tv3, INDEX(1))) &&
  19192. - !cascade(base, &base->tv4, INDEX(2)))
  19193. - cascade(base, &base->tv5, INDEX(3));
  19194. - ++base->timer_jiffies;
  19195. - hlist_move_list(base->tv1.vec + index, head);
  19196. - while (!hlist_empty(head)) {
  19197. - void (*fn)(unsigned long);
  19198. - unsigned long data;
  19199. - bool irqsafe;
  19200. -
  19201. - timer = hlist_entry(head->first, struct timer_list, entry);
  19202. - fn = timer->function;
  19203. - data = timer->data;
  19204. - irqsafe = timer->flags & TIMER_IRQSAFE;
  19205. -
  19206. - timer_stats_account_timer(timer);
  19207. -
  19208. - base->running_timer = timer;
  19209. - detach_expired_timer(timer, base);
  19210. -
  19211. - if (irqsafe) {
  19212. - spin_unlock(&base->lock);
  19213. - call_timer_fn(timer, fn, data);
  19214. - spin_lock(&base->lock);
  19215. - } else {
  19216. - spin_unlock_irq(&base->lock);
  19217. - call_timer_fn(timer, fn, data);
  19218. - spin_lock_irq(&base->lock);
  19219. - }
  19220. + for (i = 0; i < LVL_DEPTH; i++) {
  19221. + idx = (clk & LVL_MASK) + i * LVL_SIZE;
  19222. +
  19223. + if (__test_and_clear_bit(idx, base->pending_map)) {
  19224. + vec = base->vectors + idx;
  19225. + hlist_move_list(vec, heads++);
  19226. + levels++;
  19227. }
  19228. + /* Is it time to look at the next level? */
  19229. + if (clk & LVL_CLK_MASK)
  19230. + break;
  19231. + /* Shift clock for the next level granularity */
  19232. + clk >>= LVL_CLK_SHIFT;
  19233. }
  19234. - base->running_timer = NULL;
  19235. - spin_unlock_irq(&base->lock);
  19236. + return levels;
  19237. }
  19238. #ifdef CONFIG_NO_HZ_COMMON
  19239. /*
  19240. - * Find out when the next timer event is due to happen. This
  19241. - * is used on S/390 to stop all activity when a CPU is idle.
  19242. - * This function needs to be called with interrupts disabled.
  19243. - */
  19244. -static unsigned long __next_timer_interrupt(struct tvec_base *base)
  19245. -{
  19246. - unsigned long timer_jiffies = base->timer_jiffies;
  19247. - unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA;
  19248. - int index, slot, array, found = 0;
  19249. - struct timer_list *nte;
  19250. - struct tvec *varray[4];
  19251. -
  19252. - /* Look for timer events in tv1. */
  19253. - index = slot = timer_jiffies & TVR_MASK;
  19254. - do {
  19255. - hlist_for_each_entry(nte, base->tv1.vec + slot, entry) {
  19256. - if (nte->flags & TIMER_DEFERRABLE)
  19257. - continue;
  19258. -
  19259. - found = 1;
  19260. - expires = nte->expires;
  19261. - /* Look at the cascade bucket(s)? */
  19262. - if (!index || slot < index)
  19263. - goto cascade;
  19264. - return expires;
  19265. - }
  19266. - slot = (slot + 1) & TVR_MASK;
  19267. - } while (slot != index);
  19268. + * Find the next pending bucket of a level. Search from level start (@offset)
  19269. + * + @clk upwards and if nothing there, search from start of the level
  19270. + * (@offset) up to @offset + clk.
  19271. + */
  19272. +static int next_pending_bucket(struct timer_base *base, unsigned offset,
  19273. + unsigned clk)
  19274. +{
  19275. + unsigned pos, start = offset + clk;
  19276. + unsigned end = offset + LVL_SIZE;
  19277. +
  19278. + pos = find_next_bit(base->pending_map, end, start);
  19279. + if (pos < end)
  19280. + return pos - start;
  19281. -cascade:
  19282. - /* Calculate the next cascade event */
  19283. - if (index)
  19284. - timer_jiffies += TVR_SIZE - index;
  19285. - timer_jiffies >>= TVR_BITS;
  19286. -
  19287. - /* Check tv2-tv5. */
  19288. - varray[0] = &base->tv2;
  19289. - varray[1] = &base->tv3;
  19290. - varray[2] = &base->tv4;
  19291. - varray[3] = &base->tv5;
  19292. -
  19293. - for (array = 0; array < 4; array++) {
  19294. - struct tvec *varp = varray[array];
  19295. -
  19296. - index = slot = timer_jiffies & TVN_MASK;
  19297. - do {
  19298. - hlist_for_each_entry(nte, varp->vec + slot, entry) {
  19299. - if (nte->flags & TIMER_DEFERRABLE)
  19300. - continue;
  19301. -
  19302. - found = 1;
  19303. - if (time_before(nte->expires, expires))
  19304. - expires = nte->expires;
  19305. - }
  19306. - /*
  19307. - * Do we still search for the first timer or are
  19308. - * we looking up the cascade buckets ?
  19309. - */
  19310. - if (found) {
  19311. - /* Look at the cascade bucket(s)? */
  19312. - if (!index || slot < index)
  19313. - break;
  19314. - return expires;
  19315. - }
  19316. - slot = (slot + 1) & TVN_MASK;
  19317. - } while (slot != index);
  19318. -
  19319. - if (index)
  19320. - timer_jiffies += TVN_SIZE - index;
  19321. - timer_jiffies >>= TVN_BITS;
  19322. + pos = find_next_bit(base->pending_map, start, offset);
  19323. + return pos < start ? pos + LVL_SIZE - start : -1;
  19324. +}
  19325. +
  19326. +/*
  19327. + * Search the first expiring timer in the various clock levels. Caller must
  19328. + * hold base->lock.
  19329. + */
  19330. +static unsigned long __next_timer_interrupt(struct timer_base *base)
  19331. +{
  19332. + unsigned long clk, next, adj;
  19333. + unsigned lvl, offset = 0;
  19334. +
  19335. + next = base->clk + NEXT_TIMER_MAX_DELTA;
  19336. + clk = base->clk;
  19337. + for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) {
  19338. + int pos = next_pending_bucket(base, offset, clk & LVL_MASK);
  19339. +
  19340. + if (pos >= 0) {
  19341. + unsigned long tmp = clk + (unsigned long) pos;
  19342. +
  19343. + tmp <<= LVL_SHIFT(lvl);
  19344. + if (time_before(tmp, next))
  19345. + next = tmp;
  19346. + }
  19347. + /*
  19348. + * Clock for the next level. If the current level clock lower
  19349. + * bits are zero, we look at the next level as is. If not we
  19350. + * need to advance it by one because that's going to be the
  19351. + * next expiring bucket in that level. base->clk is the next
  19352. + * expiring jiffie. So in case of:
  19353. + *
  19354. + * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
  19355. + * 0 0 0 0 0 0
  19356. + *
  19357. + * we have to look at all levels @index 0. With
  19358. + *
  19359. + * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
  19360. + * 0 0 0 0 0 2
  19361. + *
  19362. + * LVL0 has the next expiring bucket @index 2. The upper
  19363. + * levels have the next expiring bucket @index 1.
  19364. + *
  19365. + * In case that the propagation wraps the next level the same
  19366. + * rules apply:
  19367. + *
  19368. + * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
  19369. + * 0 0 0 0 F 2
  19370. + *
  19371. + * So after looking at LVL0 we get:
  19372. + *
  19373. + * LVL5 LVL4 LVL3 LVL2 LVL1
  19374. + * 0 0 0 1 0
  19375. + *
  19376. + * So no propagation from LVL1 to LVL2 because that happened
  19377. + * with the add already, but then we need to propagate further
  19378. + * from LVL2 to LVL3.
  19379. + *
  19380. + * So the simple check whether the lower bits of the current
  19381. + * level are 0 or not is sufficient for all cases.
  19382. + */
  19383. + adj = clk & LVL_CLK_MASK ? 1 : 0;
  19384. + clk >>= LVL_CLK_SHIFT;
  19385. + clk += adj;
  19386. }
  19387. - return expires;
  19388. + return next;
  19389. }
  19390. /*
  19391. @@ -1379,7 +1541,7 @@
  19392. */
  19393. u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
  19394. {
  19395. - struct tvec_base *base = this_cpu_ptr(&tvec_bases);
  19396. + struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  19397. u64 expires = KTIME_MAX;
  19398. unsigned long nextevt;
  19399. @@ -1390,20 +1552,81 @@
  19400. if (cpu_is_offline(smp_processor_id()))
  19401. return expires;
  19402. - spin_lock(&base->lock);
  19403. - if (base->active_timers) {
  19404. - if (time_before_eq(base->next_timer, base->timer_jiffies))
  19405. - base->next_timer = __next_timer_interrupt(base);
  19406. - nextevt = base->next_timer;
  19407. - if (time_before_eq(nextevt, basej))
  19408. - expires = basem;
  19409. - else
  19410. - expires = basem + (nextevt - basej) * TICK_NSEC;
  19411. + raw_spin_lock(&base->lock);
  19412. + nextevt = __next_timer_interrupt(base);
  19413. + base->next_expiry = nextevt;
  19414. + /*
  19415. + * We have a fresh next event. Check whether we can forward the base.
  19416. + */
  19417. + if (time_after(nextevt, jiffies))
  19418. + base->clk = jiffies;
  19419. + else if (time_after(nextevt, base->clk))
  19420. + base->clk = nextevt;
  19421. +
  19422. + if (time_before_eq(nextevt, basej)) {
  19423. + expires = basem;
  19424. + base->is_idle = false;
  19425. + } else {
  19426. + expires = basem + (nextevt - basej) * TICK_NSEC;
  19427. + /*
  19428. + * If we expect to sleep more than a tick, mark the base idle.
  19429. + */
  19430. + if ((expires - basem) > TICK_NSEC)
  19431. + base->is_idle = true;
  19432. }
  19433. - spin_unlock(&base->lock);
  19434. + raw_spin_unlock(&base->lock);
  19435. return cmp_next_hrtimer_event(basem, expires);
  19436. }
  19437. +
  19438. +/**
  19439. + * timer_clear_idle - Clear the idle state of the timer base
  19440. + *
  19441. + * Called with interrupts disabled
  19442. + */
  19443. +void timer_clear_idle(void)
  19444. +{
  19445. + struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  19446. +
  19447. + /*
  19448. + * We do this unlocked. The worst outcome is a remote enqueue sending
  19449. + * a pointless IPI, but taking the lock would just make the window for
  19450. + * sending the IPI a few instructions smaller for the cost of taking
  19451. + * the lock in the exit from idle path.
  19452. + */
  19453. + base->is_idle = false;
  19454. +}
  19455. +
  19456. +static int collect_expired_timers(struct timer_base *base,
  19457. + struct hlist_head *heads)
  19458. +{
  19459. + /*
  19460. + * NOHZ optimization. After a long idle sleep we need to forward the
  19461. + * base to current jiffies. Avoid a loop by searching the bitfield for
  19462. + * the next expiring timer.
  19463. + */
  19464. + if ((long)(jiffies - base->clk) > 2) {
  19465. + unsigned long next = __next_timer_interrupt(base);
  19466. +
  19467. + /*
  19468. + * If the next timer is ahead of time forward to current
  19469. + * jiffies, otherwise forward to the next expiry time.
  19470. + */
  19471. + if (time_after(next, jiffies)) {
  19472. + /* The call site will increment clock! */
  19473. + base->clk = jiffies - 1;
  19474. + return 0;
  19475. + }
  19476. + base->clk = next;
  19477. + }
  19478. + return __collect_expired_timers(base, heads);
  19479. +}
  19480. +#else
  19481. +static inline int collect_expired_timers(struct timer_base *base,
  19482. + struct hlist_head *heads)
  19483. +{
  19484. + return __collect_expired_timers(base, heads);
  19485. +}
  19486. #endif
  19487. /*
  19488. @@ -1416,25 +1639,54 @@
  19489. /* Note: this timer irq context must be accounted for as well. */
  19490. account_process_tick(p, user_tick);
  19491. + scheduler_tick();
  19492. run_local_timers();
  19493. rcu_check_callbacks(user_tick);
  19494. -#ifdef CONFIG_IRQ_WORK
  19495. +#if defined(CONFIG_IRQ_WORK)
  19496. if (in_irq())
  19497. irq_work_tick();
  19498. #endif
  19499. - scheduler_tick();
  19500. run_posix_cpu_timers(p);
  19501. }
  19502. +/**
  19503. + * __run_timers - run all expired timers (if any) on this CPU.
  19504. + * @base: the timer vector to be processed.
  19505. + */
  19506. +static inline void __run_timers(struct timer_base *base)
  19507. +{
  19508. + struct hlist_head heads[LVL_DEPTH];
  19509. + int levels;
  19510. +
  19511. + if (!time_after_eq(jiffies, base->clk))
  19512. + return;
  19513. +
  19514. + raw_spin_lock_irq(&base->lock);
  19515. +
  19516. + while (time_after_eq(jiffies, base->clk)) {
  19517. +
  19518. + levels = collect_expired_timers(base, heads);
  19519. + base->clk++;
  19520. +
  19521. + while (levels--)
  19522. + expire_timers(base, heads + levels);
  19523. + }
  19524. + raw_spin_unlock_irq(&base->lock);
  19525. + wakeup_timer_waiters(base);
  19526. +}
  19527. +
  19528. /*
  19529. * This function runs timers and the timer-tq in bottom half context.
  19530. */
  19531. static void run_timer_softirq(struct softirq_action *h)
  19532. {
  19533. - struct tvec_base *base = this_cpu_ptr(&tvec_bases);
  19534. + struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  19535. +
  19536. + irq_work_tick_soft();
  19537. - if (time_after_eq(jiffies, base->timer_jiffies))
  19538. - __run_timers(base);
  19539. + __run_timers(base);
  19540. + if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active)
  19541. + __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
  19542. }
  19543. /*
  19544. @@ -1442,7 +1694,18 @@
  19545. */
  19546. void run_local_timers(void)
  19547. {
  19548. + struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  19549. +
  19550. hrtimer_run_queues();
  19551. + /* Raise the softirq only if required. */
  19552. + if (time_before(jiffies, base->clk)) {
  19553. + if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active)
  19554. + return;
  19555. + /* CPU is awake, so check the deferrable base. */
  19556. + base++;
  19557. + if (time_before(jiffies, base->clk))
  19558. + return;
  19559. + }
  19560. raise_softirq(TIMER_SOFTIRQ);
  19561. }
  19562. @@ -1527,7 +1790,7 @@
  19563. expire = timeout + jiffies;
  19564. setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
  19565. - __mod_timer(&timer, expire, false, TIMER_NOT_PINNED);
  19566. + __mod_timer(&timer, expire, false);
  19567. schedule();
  19568. del_singleshot_timer_sync(&timer);
  19569. @@ -1578,14 +1841,13 @@
  19570. EXPORT_SYMBOL(schedule_timeout_idle);
  19571. #ifdef CONFIG_HOTPLUG_CPU
  19572. -static void migrate_timer_list(struct tvec_base *new_base, struct hlist_head *head)
  19573. +static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head)
  19574. {
  19575. struct timer_list *timer;
  19576. int cpu = new_base->cpu;
  19577. while (!hlist_empty(head)) {
  19578. timer = hlist_entry(head->first, struct timer_list, entry);
  19579. - /* We ignore the accounting on the dying cpu */
  19580. detach_timer(timer, false);
  19581. timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;
  19582. internal_add_timer(new_base, timer);
  19583. @@ -1594,37 +1856,31 @@
  19584. static void migrate_timers(int cpu)
  19585. {
  19586. - struct tvec_base *old_base;
  19587. - struct tvec_base *new_base;
  19588. - int i;
  19589. + struct timer_base *old_base;
  19590. + struct timer_base *new_base;
  19591. + int b, i;
  19592. BUG_ON(cpu_online(cpu));
  19593. - old_base = per_cpu_ptr(&tvec_bases, cpu);
  19594. - new_base = get_cpu_ptr(&tvec_bases);
  19595. - /*
  19596. - * The caller is globally serialized and nobody else
  19597. - * takes two locks at once, deadlock is not possible.
  19598. - */
  19599. - spin_lock_irq(&new_base->lock);
  19600. - spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
  19601. - BUG_ON(old_base->running_timer);
  19602. + for (b = 0; b < NR_BASES; b++) {
  19603. + old_base = per_cpu_ptr(&timer_bases[b], cpu);
  19604. + new_base = get_cpu_ptr(&timer_bases[b]);
  19605. + /*
  19606. + * The caller is globally serialized and nobody else
  19607. + * takes two locks at once, deadlock is not possible.
  19608. + */
  19609. + raw_spin_lock_irq(&new_base->lock);
  19610. + raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
  19611. - for (i = 0; i < TVR_SIZE; i++)
  19612. - migrate_timer_list(new_base, old_base->tv1.vec + i);
  19613. - for (i = 0; i < TVN_SIZE; i++) {
  19614. - migrate_timer_list(new_base, old_base->tv2.vec + i);
  19615. - migrate_timer_list(new_base, old_base->tv3.vec + i);
  19616. - migrate_timer_list(new_base, old_base->tv4.vec + i);
  19617. - migrate_timer_list(new_base, old_base->tv5.vec + i);
  19618. - }
  19619. + BUG_ON(old_base->running_timer);
  19620. - old_base->active_timers = 0;
  19621. - old_base->all_timers = 0;
  19622. + for (i = 0; i < WHEEL_SIZE; i++)
  19623. + migrate_timer_list(new_base, old_base->vectors + i);
  19624. - spin_unlock(&old_base->lock);
  19625. - spin_unlock_irq(&new_base->lock);
  19626. - put_cpu_ptr(&tvec_bases);
  19627. + raw_spin_unlock(&old_base->lock);
  19628. + raw_spin_unlock_irq(&new_base->lock);
  19629. + put_cpu_ptr(&timer_bases);
  19630. + }
  19631. }
  19632. static int timer_cpu_notify(struct notifier_block *self,
  19633. @@ -1652,13 +1908,18 @@
  19634. static void __init init_timer_cpu(int cpu)
  19635. {
  19636. - struct tvec_base *base = per_cpu_ptr(&tvec_bases, cpu);
  19637. -
  19638. - base->cpu = cpu;
  19639. - spin_lock_init(&base->lock);
  19640. + struct timer_base *base;
  19641. + int i;
  19642. - base->timer_jiffies = jiffies;
  19643. - base->next_timer = base->timer_jiffies;
  19644. + for (i = 0; i < NR_BASES; i++) {
  19645. + base = per_cpu_ptr(&timer_bases[i], cpu);
  19646. + base->cpu = cpu;
  19647. + raw_spin_lock_init(&base->lock);
  19648. + base->clk = jiffies;
  19649. +#ifdef CONFIG_PREEMPT_RT_FULL
  19650. + init_swait_queue_head(&base->wait_for_running_timer);
  19651. +#endif
  19652. + }
  19653. }
  19654. static void __init init_timer_cpus(void)
  19655. diff -Nur linux-4.6.6.orig/kernel/trace/Kconfig linux-4.6.6/kernel/trace/Kconfig
  19656. --- linux-4.6.6.orig/kernel/trace/Kconfig 2016-08-10 12:54:57.000000000 +0200
  19657. +++ linux-4.6.6/kernel/trace/Kconfig 2016-08-22 21:27:47.425893881 +0200
  19658. @@ -187,6 +187,24 @@
  19659. enabled. This option and the preempt-off timing option can be
  19660. used together or separately.)
  19661. +config INTERRUPT_OFF_HIST
  19662. + bool "Interrupts-off Latency Histogram"
  19663. + depends on IRQSOFF_TRACER
  19664. + help
  19665. + This option generates continuously updated histograms (one per cpu)
  19666. + of the duration of time periods with interrupts disabled. The
  19667. + histograms are disabled by default. To enable them, write a non-zero
  19668. + number to
  19669. +
  19670. + /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff
  19671. +
  19672. + If PREEMPT_OFF_HIST is also selected, additional histograms (one
  19673. + per cpu) are generated that accumulate the duration of time periods
  19674. + when both interrupts and preemption are disabled. The histogram data
  19675. + will be located in the debug file system at
  19676. +
  19677. + /sys/kernel/debug/tracing/latency_hist/irqsoff
  19678. +
  19679. config PREEMPT_TRACER
  19680. bool "Preemption-off Latency Tracer"
  19681. default n
  19682. @@ -211,6 +229,24 @@
  19683. enabled. This option and the irqs-off timing option can be
  19684. used together or separately.)
  19685. +config PREEMPT_OFF_HIST
  19686. + bool "Preemption-off Latency Histogram"
  19687. + depends on PREEMPT_TRACER
  19688. + help
  19689. + This option generates continuously updated histograms (one per cpu)
  19690. + of the duration of time periods with preemption disabled. The
  19691. + histograms are disabled by default. To enable them, write a non-zero
  19692. + number to
  19693. +
  19694. + /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff
  19695. +
  19696. + If INTERRUPT_OFF_HIST is also selected, additional histograms (one
  19697. + per cpu) are generated that accumulate the duration of time periods
  19698. + when both interrupts and preemption are disabled. The histogram data
  19699. + will be located in the debug file system at
  19700. +
  19701. + /sys/kernel/debug/tracing/latency_hist/preemptoff
  19702. +
  19703. config SCHED_TRACER
  19704. bool "Scheduling Latency Tracer"
  19705. select GENERIC_TRACER
  19706. @@ -221,6 +257,74 @@
  19707. This tracer tracks the latency of the highest priority task
  19708. to be scheduled in, starting from the point it has woken up.
  19709. +config WAKEUP_LATENCY_HIST
  19710. + bool "Scheduling Latency Histogram"
  19711. + depends on SCHED_TRACER
  19712. + help
  19713. + This option generates continuously updated histograms (one per cpu)
  19714. + of the scheduling latency of the highest priority task.
  19715. + The histograms are disabled by default. To enable them, write a
  19716. + non-zero number to
  19717. +
  19718. + /sys/kernel/debug/tracing/latency_hist/enable/wakeup
  19719. +
  19720. + Two different algorithms are used, one to determine the latency of
  19721. + processes that exclusively use the highest priority of the system and
  19722. + another one to determine the latency of processes that share the
  19723. + highest system priority with other processes. The former is used to
  19724. + improve hardware and system software, the latter to optimize the
  19725. + priority design of a given system. The histogram data will be
  19726. + located in the debug file system at
  19727. +
  19728. + /sys/kernel/debug/tracing/latency_hist/wakeup
  19729. +
  19730. + and
  19731. +
  19732. + /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio
  19733. +
  19734. + If both Scheduling Latency Histogram and Missed Timer Offsets
  19735. + Histogram are selected, additional histogram data will be collected
  19736. + that contain, in addition to the wakeup latency, the timer latency, in
  19737. + case the wakeup was triggered by an expired timer. These histograms
  19738. + are available in the
  19739. +
  19740. + /sys/kernel/debug/tracing/latency_hist/timerandwakeup
  19741. +
  19742. + directory. They reflect the apparent interrupt and scheduling latency
  19743. + and are best suitable to determine the worst-case latency of a given
  19744. + system. To enable these histograms, write a non-zero number to
  19745. +
  19746. + /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup
  19747. +
  19748. +config MISSED_TIMER_OFFSETS_HIST
  19749. + depends on HIGH_RES_TIMERS
  19750. + select GENERIC_TRACER
  19751. + bool "Missed Timer Offsets Histogram"
  19752. + help
  19753. + Generate a histogram of missed timer offsets in microseconds. The
  19754. + histograms are disabled by default. To enable them, write a non-zero
  19755. + number to
  19756. +
  19757. + /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets
  19758. +
  19759. + The histogram data will be located in the debug file system at
  19760. +
  19761. + /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets
  19762. +
  19763. + If both Scheduling Latency Histogram and Missed Timer Offsets
  19764. + Histogram are selected, additional histogram data will be collected
  19765. + that contain, in addition to the wakeup latency, the timer latency, in
  19766. + case the wakeup was triggered by an expired timer. These histograms
  19767. + are available in the
  19768. +
  19769. + /sys/kernel/debug/tracing/latency_hist/timerandwakeup
  19770. +
  19771. + directory. They reflect the apparent interrupt and scheduling latency
  19772. + and are best suitable to determine the worst-case latency of a given
  19773. + system. To enable these histograms, write a non-zero number to
  19774. +
  19775. + /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup
  19776. +
  19777. config ENABLE_DEFAULT_TRACERS
  19778. bool "Trace process context switches and events"
  19779. depends on !GENERIC_TRACER
  19780. diff -Nur linux-4.6.6.orig/kernel/trace/latency_hist.c linux-4.6.6/kernel/trace/latency_hist.c
  19781. --- linux-4.6.6.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100
  19782. +++ linux-4.6.6/kernel/trace/latency_hist.c 2016-08-22 21:27:47.425893881 +0200
  19783. @@ -0,0 +1,1178 @@
  19784. +/*
  19785. + * kernel/trace/latency_hist.c
  19786. + *
  19787. + * Add support for histograms of preemption-off latency and
  19788. + * interrupt-off latency and wakeup latency, it depends on
  19789. + * Real-Time Preemption Support.
  19790. + *
  19791. + * Copyright (C) 2005 MontaVista Software, Inc.
  19792. + * Yi Yang <yyang@ch.mvista.com>
  19793. + *
  19794. + * Converted to work with the new latency tracer.
  19795. + * Copyright (C) 2008 Red Hat, Inc.
  19796. + * Steven Rostedt <srostedt@redhat.com>
  19797. + *
  19798. + */
  19799. +#include <linux/module.h>
  19800. +#include <linux/debugfs.h>
  19801. +#include <linux/seq_file.h>
  19802. +#include <linux/percpu.h>
  19803. +#include <linux/kallsyms.h>
  19804. +#include <linux/uaccess.h>
  19805. +#include <linux/sched.h>
  19806. +#include <linux/sched/rt.h>
  19807. +#include <linux/slab.h>
  19808. +#include <linux/atomic.h>
  19809. +#include <asm/div64.h>
  19810. +
  19811. +#include "trace.h"
  19812. +#include <trace/events/sched.h>
  19813. +
  19814. +#define NSECS_PER_USECS 1000L
  19815. +
  19816. +#define CREATE_TRACE_POINTS
  19817. +#include <trace/events/hist.h>
  19818. +
  19819. +enum {
  19820. + IRQSOFF_LATENCY = 0,
  19821. + PREEMPTOFF_LATENCY,
  19822. + PREEMPTIRQSOFF_LATENCY,
  19823. + WAKEUP_LATENCY,
  19824. + WAKEUP_LATENCY_SHAREDPRIO,
  19825. + MISSED_TIMER_OFFSETS,
  19826. + TIMERANDWAKEUP_LATENCY,
  19827. + MAX_LATENCY_TYPE,
  19828. +};
  19829. +
  19830. +#define MAX_ENTRY_NUM 10240
  19831. +
  19832. +struct hist_data {
  19833. + atomic_t hist_mode; /* 0 log, 1 don't log */
  19834. + long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */
  19835. + long min_lat;
  19836. + long max_lat;
  19837. + unsigned long long below_hist_bound_samples;
  19838. + unsigned long long above_hist_bound_samples;
  19839. + long long accumulate_lat;
  19840. + unsigned long long total_samples;
  19841. + unsigned long long hist_array[MAX_ENTRY_NUM];
  19842. +};
  19843. +
  19844. +struct enable_data {
  19845. + int latency_type;
  19846. + int enabled;
  19847. +};
  19848. +
  19849. +static char *latency_hist_dir_root = "latency_hist";
  19850. +
  19851. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19852. +static DEFINE_PER_CPU(struct hist_data, irqsoff_hist);
  19853. +static char *irqsoff_hist_dir = "irqsoff";
  19854. +static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start);
  19855. +static DEFINE_PER_CPU(int, hist_irqsoff_counting);
  19856. +#endif
  19857. +
  19858. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19859. +static DEFINE_PER_CPU(struct hist_data, preemptoff_hist);
  19860. +static char *preemptoff_hist_dir = "preemptoff";
  19861. +static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start);
  19862. +static DEFINE_PER_CPU(int, hist_preemptoff_counting);
  19863. +#endif
  19864. +
  19865. +#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST)
  19866. +static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist);
  19867. +static char *preemptirqsoff_hist_dir = "preemptirqsoff";
  19868. +static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start);
  19869. +static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting);
  19870. +#endif
  19871. +
  19872. +#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST)
  19873. +static notrace void probe_preemptirqsoff_hist(void *v, int reason, int start);
  19874. +static struct enable_data preemptirqsoff_enabled_data = {
  19875. + .latency_type = PREEMPTIRQSOFF_LATENCY,
  19876. + .enabled = 0,
  19877. +};
  19878. +#endif
  19879. +
  19880. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19881. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19882. +struct maxlatproc_data {
  19883. + char comm[FIELD_SIZEOF(struct task_struct, comm)];
  19884. + char current_comm[FIELD_SIZEOF(struct task_struct, comm)];
  19885. + int pid;
  19886. + int current_pid;
  19887. + int prio;
  19888. + int current_prio;
  19889. + long latency;
  19890. + long timeroffset;
  19891. + cycle_t timestamp;
  19892. +};
  19893. +#endif
  19894. +
  19895. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19896. +static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist);
  19897. +static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio);
  19898. +static char *wakeup_latency_hist_dir = "wakeup";
  19899. +static char *wakeup_latency_hist_dir_sharedprio = "sharedprio";
  19900. +static notrace void probe_wakeup_latency_hist_start(void *v,
  19901. + struct task_struct *p);
  19902. +static notrace void probe_wakeup_latency_hist_stop(void *v,
  19903. + bool preempt, struct task_struct *prev, struct task_struct *next);
  19904. +static notrace void probe_sched_migrate_task(void *,
  19905. + struct task_struct *task, int cpu);
  19906. +static struct enable_data wakeup_latency_enabled_data = {
  19907. + .latency_type = WAKEUP_LATENCY,
  19908. + .enabled = 0,
  19909. +};
  19910. +static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc);
  19911. +static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio);
  19912. +static DEFINE_PER_CPU(struct task_struct *, wakeup_task);
  19913. +static DEFINE_PER_CPU(int, wakeup_sharedprio);
  19914. +static unsigned long wakeup_pid;
  19915. +#endif
  19916. +
  19917. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19918. +static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets);
  19919. +static char *missed_timer_offsets_dir = "missed_timer_offsets";
  19920. +static notrace void probe_hrtimer_interrupt(void *v, int cpu,
  19921. + long long offset, struct task_struct *curr, struct task_struct *task);
  19922. +static struct enable_data missed_timer_offsets_enabled_data = {
  19923. + .latency_type = MISSED_TIMER_OFFSETS,
  19924. + .enabled = 0,
  19925. +};
  19926. +static DEFINE_PER_CPU(struct maxlatproc_data, missed_timer_offsets_maxlatproc);
  19927. +static unsigned long missed_timer_offsets_pid;
  19928. +#endif
  19929. +
  19930. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  19931. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19932. +static DEFINE_PER_CPU(struct hist_data, timerandwakeup_latency_hist);
  19933. +static char *timerandwakeup_latency_hist_dir = "timerandwakeup";
  19934. +static struct enable_data timerandwakeup_enabled_data = {
  19935. + .latency_type = TIMERANDWAKEUP_LATENCY,
  19936. + .enabled = 0,
  19937. +};
  19938. +static DEFINE_PER_CPU(struct maxlatproc_data, timerandwakeup_maxlatproc);
  19939. +#endif
  19940. +
  19941. +void notrace latency_hist(int latency_type, int cpu, long latency,
  19942. + long timeroffset, cycle_t stop,
  19943. + struct task_struct *p)
  19944. +{
  19945. + struct hist_data *my_hist;
  19946. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19947. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19948. + struct maxlatproc_data *mp = NULL;
  19949. +#endif
  19950. +
  19951. + if (!cpu_possible(cpu) || latency_type < 0 ||
  19952. + latency_type >= MAX_LATENCY_TYPE)
  19953. + return;
  19954. +
  19955. + switch (latency_type) {
  19956. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19957. + case IRQSOFF_LATENCY:
  19958. + my_hist = &per_cpu(irqsoff_hist, cpu);
  19959. + break;
  19960. +#endif
  19961. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19962. + case PREEMPTOFF_LATENCY:
  19963. + my_hist = &per_cpu(preemptoff_hist, cpu);
  19964. + break;
  19965. +#endif
  19966. +#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST)
  19967. + case PREEMPTIRQSOFF_LATENCY:
  19968. + my_hist = &per_cpu(preemptirqsoff_hist, cpu);
  19969. + break;
  19970. +#endif
  19971. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19972. + case WAKEUP_LATENCY:
  19973. + my_hist = &per_cpu(wakeup_latency_hist, cpu);
  19974. + mp = &per_cpu(wakeup_maxlatproc, cpu);
  19975. + break;
  19976. + case WAKEUP_LATENCY_SHAREDPRIO:
  19977. + my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu);
  19978. + mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu);
  19979. + break;
  19980. +#endif
  19981. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19982. + case MISSED_TIMER_OFFSETS:
  19983. + my_hist = &per_cpu(missed_timer_offsets, cpu);
  19984. + mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu);
  19985. + break;
  19986. +#endif
  19987. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  19988. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19989. + case TIMERANDWAKEUP_LATENCY:
  19990. + my_hist = &per_cpu(timerandwakeup_latency_hist, cpu);
  19991. + mp = &per_cpu(timerandwakeup_maxlatproc, cpu);
  19992. + break;
  19993. +#endif
  19994. +
  19995. + default:
  19996. + return;
  19997. + }
  19998. +
  19999. + latency += my_hist->offset;
  20000. +
  20001. + if (atomic_read(&my_hist->hist_mode) == 0)
  20002. + return;
  20003. +
  20004. + if (latency < 0 || latency >= MAX_ENTRY_NUM) {
  20005. + if (latency < 0)
  20006. + my_hist->below_hist_bound_samples++;
  20007. + else
  20008. + my_hist->above_hist_bound_samples++;
  20009. + } else
  20010. + my_hist->hist_array[latency]++;
  20011. +
  20012. + if (unlikely(latency > my_hist->max_lat ||
  20013. + my_hist->min_lat == LONG_MAX)) {
  20014. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20015. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20016. + if (latency_type == WAKEUP_LATENCY ||
  20017. + latency_type == WAKEUP_LATENCY_SHAREDPRIO ||
  20018. + latency_type == MISSED_TIMER_OFFSETS ||
  20019. + latency_type == TIMERANDWAKEUP_LATENCY) {
  20020. + strncpy(mp->comm, p->comm, sizeof(mp->comm));
  20021. + strncpy(mp->current_comm, current->comm,
  20022. + sizeof(mp->current_comm));
  20023. + mp->pid = task_pid_nr(p);
  20024. + mp->current_pid = task_pid_nr(current);
  20025. + mp->prio = p->prio;
  20026. + mp->current_prio = current->prio;
  20027. + mp->latency = latency;
  20028. + mp->timeroffset = timeroffset;
  20029. + mp->timestamp = stop;
  20030. + }
  20031. +#endif
  20032. + my_hist->max_lat = latency;
  20033. + }
  20034. + if (unlikely(latency < my_hist->min_lat))
  20035. + my_hist->min_lat = latency;
  20036. + my_hist->total_samples++;
  20037. + my_hist->accumulate_lat += latency;
  20038. +}
  20039. +
  20040. +static void *l_start(struct seq_file *m, loff_t *pos)
  20041. +{
  20042. + loff_t *index_ptr = NULL;
  20043. + loff_t index = *pos;
  20044. + struct hist_data *my_hist = m->private;
  20045. +
  20046. + if (index == 0) {
  20047. + char minstr[32], avgstr[32], maxstr[32];
  20048. +
  20049. + atomic_dec(&my_hist->hist_mode);
  20050. +
  20051. + if (likely(my_hist->total_samples)) {
  20052. + long avg = (long) div64_s64(my_hist->accumulate_lat,
  20053. + my_hist->total_samples);
  20054. + snprintf(minstr, sizeof(minstr), "%ld",
  20055. + my_hist->min_lat - my_hist->offset);
  20056. + snprintf(avgstr, sizeof(avgstr), "%ld",
  20057. + avg - my_hist->offset);
  20058. + snprintf(maxstr, sizeof(maxstr), "%ld",
  20059. + my_hist->max_lat - my_hist->offset);
  20060. + } else {
  20061. + strcpy(minstr, "<undef>");
  20062. + strcpy(avgstr, minstr);
  20063. + strcpy(maxstr, minstr);
  20064. + }
  20065. +
  20066. + seq_printf(m, "#Minimum latency: %s microseconds\n"
  20067. + "#Average latency: %s microseconds\n"
  20068. + "#Maximum latency: %s microseconds\n"
  20069. + "#Total samples: %llu\n"
  20070. + "#There are %llu samples lower than %ld"
  20071. + " microseconds.\n"
  20072. + "#There are %llu samples greater or equal"
  20073. + " than %ld microseconds.\n"
  20074. + "#usecs\t%16s\n",
  20075. + minstr, avgstr, maxstr,
  20076. + my_hist->total_samples,
  20077. + my_hist->below_hist_bound_samples,
  20078. + -my_hist->offset,
  20079. + my_hist->above_hist_bound_samples,
  20080. + MAX_ENTRY_NUM - my_hist->offset,
  20081. + "samples");
  20082. + }
  20083. + if (index < MAX_ENTRY_NUM) {
  20084. + index_ptr = kmalloc(sizeof(loff_t), GFP_KERNEL);
  20085. + if (index_ptr)
  20086. + *index_ptr = index;
  20087. + }
  20088. +
  20089. + return index_ptr;
  20090. +}
  20091. +
  20092. +static void *l_next(struct seq_file *m, void *p, loff_t *pos)
  20093. +{
  20094. + loff_t *index_ptr = p;
  20095. + struct hist_data *my_hist = m->private;
  20096. +
  20097. + if (++*pos >= MAX_ENTRY_NUM) {
  20098. + atomic_inc(&my_hist->hist_mode);
  20099. + return NULL;
  20100. + }
  20101. + *index_ptr = *pos;
  20102. + return index_ptr;
  20103. +}
  20104. +
  20105. +static void l_stop(struct seq_file *m, void *p)
  20106. +{
  20107. + kfree(p);
  20108. +}
  20109. +
  20110. +static int l_show(struct seq_file *m, void *p)
  20111. +{
  20112. + int index = *(loff_t *) p;
  20113. + struct hist_data *my_hist = m->private;
  20114. +
  20115. + seq_printf(m, "%6ld\t%16llu\n", index - my_hist->offset,
  20116. + my_hist->hist_array[index]);
  20117. + return 0;
  20118. +}
  20119. +
  20120. +static const struct seq_operations latency_hist_seq_op = {
  20121. + .start = l_start,
  20122. + .next = l_next,
  20123. + .stop = l_stop,
  20124. + .show = l_show
  20125. +};
  20126. +
  20127. +static int latency_hist_open(struct inode *inode, struct file *file)
  20128. +{
  20129. + int ret;
  20130. +
  20131. + ret = seq_open(file, &latency_hist_seq_op);
  20132. + if (!ret) {
  20133. + struct seq_file *seq = file->private_data;
  20134. + seq->private = inode->i_private;
  20135. + }
  20136. + return ret;
  20137. +}
  20138. +
  20139. +static const struct file_operations latency_hist_fops = {
  20140. + .open = latency_hist_open,
  20141. + .read = seq_read,
  20142. + .llseek = seq_lseek,
  20143. + .release = seq_release,
  20144. +};
  20145. +
  20146. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20147. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20148. +static void clear_maxlatprocdata(struct maxlatproc_data *mp)
  20149. +{
  20150. + mp->comm[0] = mp->current_comm[0] = '\0';
  20151. + mp->prio = mp->current_prio = mp->pid = mp->current_pid =
  20152. + mp->latency = mp->timeroffset = -1;
  20153. + mp->timestamp = 0;
  20154. +}
  20155. +#endif
  20156. +
  20157. +static void hist_reset(struct hist_data *hist)
  20158. +{
  20159. + atomic_dec(&hist->hist_mode);
  20160. +
  20161. + memset(hist->hist_array, 0, sizeof(hist->hist_array));
  20162. + hist->below_hist_bound_samples = 0ULL;
  20163. + hist->above_hist_bound_samples = 0ULL;
  20164. + hist->min_lat = LONG_MAX;
  20165. + hist->max_lat = LONG_MIN;
  20166. + hist->total_samples = 0ULL;
  20167. + hist->accumulate_lat = 0LL;
  20168. +
  20169. + atomic_inc(&hist->hist_mode);
  20170. +}
  20171. +
  20172. +static ssize_t
  20173. +latency_hist_reset(struct file *file, const char __user *a,
  20174. + size_t size, loff_t *off)
  20175. +{
  20176. + int cpu;
  20177. + struct hist_data *hist = NULL;
  20178. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20179. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20180. + struct maxlatproc_data *mp = NULL;
  20181. +#endif
  20182. + off_t latency_type = (off_t) file->private_data;
  20183. +
  20184. + for_each_online_cpu(cpu) {
  20185. +
  20186. + switch (latency_type) {
  20187. +#ifdef CONFIG_PREEMPT_OFF_HIST
  20188. + case PREEMPTOFF_LATENCY:
  20189. + hist = &per_cpu(preemptoff_hist, cpu);
  20190. + break;
  20191. +#endif
  20192. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  20193. + case IRQSOFF_LATENCY:
  20194. + hist = &per_cpu(irqsoff_hist, cpu);
  20195. + break;
  20196. +#endif
  20197. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  20198. + case PREEMPTIRQSOFF_LATENCY:
  20199. + hist = &per_cpu(preemptirqsoff_hist, cpu);
  20200. + break;
  20201. +#endif
  20202. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20203. + case WAKEUP_LATENCY:
  20204. + hist = &per_cpu(wakeup_latency_hist, cpu);
  20205. + mp = &per_cpu(wakeup_maxlatproc, cpu);
  20206. + break;
  20207. + case WAKEUP_LATENCY_SHAREDPRIO:
  20208. + hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu);
  20209. + mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu);
  20210. + break;
  20211. +#endif
  20212. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20213. + case MISSED_TIMER_OFFSETS:
  20214. + hist = &per_cpu(missed_timer_offsets, cpu);
  20215. + mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu);
  20216. + break;
  20217. +#endif
  20218. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  20219. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20220. + case TIMERANDWAKEUP_LATENCY:
  20221. + hist = &per_cpu(timerandwakeup_latency_hist, cpu);
  20222. + mp = &per_cpu(timerandwakeup_maxlatproc, cpu);
  20223. + break;
  20224. +#endif
  20225. + }
  20226. +
  20227. + hist_reset(hist);
  20228. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20229. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20230. + if (latency_type == WAKEUP_LATENCY ||
  20231. + latency_type == WAKEUP_LATENCY_SHAREDPRIO ||
  20232. + latency_type == MISSED_TIMER_OFFSETS ||
  20233. + latency_type == TIMERANDWAKEUP_LATENCY)
  20234. + clear_maxlatprocdata(mp);
  20235. +#endif
  20236. + }
  20237. +
  20238. + return size;
  20239. +}
  20240. +
  20241. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20242. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20243. +static ssize_t
  20244. +show_pid(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos)
  20245. +{
  20246. + char buf[64];
  20247. + int r;
  20248. + unsigned long *this_pid = file->private_data;
  20249. +
  20250. + r = snprintf(buf, sizeof(buf), "%lu\n", *this_pid);
  20251. + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  20252. +}
  20253. +
  20254. +static ssize_t do_pid(struct file *file, const char __user *ubuf,
  20255. + size_t cnt, loff_t *ppos)
  20256. +{
  20257. + char buf[64];
  20258. + unsigned long pid;
  20259. + unsigned long *this_pid = file->private_data;
  20260. +
  20261. + if (cnt >= sizeof(buf))
  20262. + return -EINVAL;
  20263. +
  20264. + if (copy_from_user(&buf, ubuf, cnt))
  20265. + return -EFAULT;
  20266. +
  20267. + buf[cnt] = '\0';
  20268. +
  20269. + if (kstrtoul(buf, 10, &pid))
  20270. + return -EINVAL;
  20271. +
  20272. + *this_pid = pid;
  20273. +
  20274. + return cnt;
  20275. +}
  20276. +#endif
  20277. +
  20278. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20279. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20280. +static ssize_t
  20281. +show_maxlatproc(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos)
  20282. +{
  20283. + int r;
  20284. + struct maxlatproc_data *mp = file->private_data;
  20285. + int strmaxlen = (TASK_COMM_LEN * 2) + (8 * 8);
  20286. + unsigned long long t;
  20287. + unsigned long usecs, secs;
  20288. + char *buf;
  20289. +
  20290. + if (mp->pid == -1 || mp->current_pid == -1) {
  20291. + buf = "(none)\n";
  20292. + return simple_read_from_buffer(ubuf, cnt, ppos, buf,
  20293. + strlen(buf));
  20294. + }
  20295. +
  20296. + buf = kmalloc(strmaxlen, GFP_KERNEL);
  20297. + if (buf == NULL)
  20298. + return -ENOMEM;
  20299. +
  20300. + t = ns2usecs(mp->timestamp);
  20301. + usecs = do_div(t, USEC_PER_SEC);
  20302. + secs = (unsigned long) t;
  20303. + r = snprintf(buf, strmaxlen,
  20304. + "%d %d %ld (%ld) %s <- %d %d %s %lu.%06lu\n", mp->pid,
  20305. + MAX_RT_PRIO-1 - mp->prio, mp->latency, mp->timeroffset, mp->comm,
  20306. + mp->current_pid, MAX_RT_PRIO-1 - mp->current_prio, mp->current_comm,
  20307. + secs, usecs);
  20308. + r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  20309. + kfree(buf);
  20310. + return r;
  20311. +}
  20312. +#endif
  20313. +
  20314. +static ssize_t
  20315. +show_enable(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos)
  20316. +{
  20317. + char buf[64];
  20318. + struct enable_data *ed = file->private_data;
  20319. + int r;
  20320. +
  20321. + r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled);
  20322. + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  20323. +}
  20324. +
  20325. +static ssize_t
  20326. +do_enable(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos)
  20327. +{
  20328. + char buf[64];
  20329. + long enable;
  20330. + struct enable_data *ed = file->private_data;
  20331. +
  20332. + if (cnt >= sizeof(buf))
  20333. + return -EINVAL;
  20334. +
  20335. + if (copy_from_user(&buf, ubuf, cnt))
  20336. + return -EFAULT;
  20337. +
  20338. + buf[cnt] = 0;
  20339. +
  20340. + if (kstrtoul(buf, 10, &enable))
  20341. + return -EINVAL;
  20342. +
  20343. + if ((enable && ed->enabled) || (!enable && !ed->enabled))
  20344. + return cnt;
  20345. +
  20346. + if (enable) {
  20347. + int ret;
  20348. +
  20349. + switch (ed->latency_type) {
  20350. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  20351. + case PREEMPTIRQSOFF_LATENCY:
  20352. + ret = register_trace_preemptirqsoff_hist(
  20353. + probe_preemptirqsoff_hist, NULL);
  20354. + if (ret) {
  20355. + pr_info("wakeup trace: Couldn't assign "
  20356. + "probe_preemptirqsoff_hist "
  20357. + "to trace_preemptirqsoff_hist\n");
  20358. + return ret;
  20359. + }
  20360. + break;
  20361. +#endif
  20362. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20363. + case WAKEUP_LATENCY:
  20364. + ret = register_trace_sched_wakeup(
  20365. + probe_wakeup_latency_hist_start, NULL);
  20366. + if (ret) {
  20367. + pr_info("wakeup trace: Couldn't assign "
  20368. + "probe_wakeup_latency_hist_start "
  20369. + "to trace_sched_wakeup\n");
  20370. + return ret;
  20371. + }
  20372. + ret = register_trace_sched_wakeup_new(
  20373. + probe_wakeup_latency_hist_start, NULL);
  20374. + if (ret) {
  20375. + pr_info("wakeup trace: Couldn't assign "
  20376. + "probe_wakeup_latency_hist_start "
  20377. + "to trace_sched_wakeup_new\n");
  20378. + unregister_trace_sched_wakeup(
  20379. + probe_wakeup_latency_hist_start, NULL);
  20380. + return ret;
  20381. + }
  20382. + ret = register_trace_sched_switch(
  20383. + probe_wakeup_latency_hist_stop, NULL);
  20384. + if (ret) {
  20385. + pr_info("wakeup trace: Couldn't assign "
  20386. + "probe_wakeup_latency_hist_stop "
  20387. + "to trace_sched_switch\n");
  20388. + unregister_trace_sched_wakeup(
  20389. + probe_wakeup_latency_hist_start, NULL);
  20390. + unregister_trace_sched_wakeup_new(
  20391. + probe_wakeup_latency_hist_start, NULL);
  20392. + return ret;
  20393. + }
  20394. + ret = register_trace_sched_migrate_task(
  20395. + probe_sched_migrate_task, NULL);
  20396. + if (ret) {
  20397. + pr_info("wakeup trace: Couldn't assign "
  20398. + "probe_sched_migrate_task "
  20399. + "to trace_sched_migrate_task\n");
  20400. + unregister_trace_sched_wakeup(
  20401. + probe_wakeup_latency_hist_start, NULL);
  20402. + unregister_trace_sched_wakeup_new(
  20403. + probe_wakeup_latency_hist_start, NULL);
  20404. + unregister_trace_sched_switch(
  20405. + probe_wakeup_latency_hist_stop, NULL);
  20406. + return ret;
  20407. + }
  20408. + break;
  20409. +#endif
  20410. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20411. + case MISSED_TIMER_OFFSETS:
  20412. + ret = register_trace_hrtimer_interrupt(
  20413. + probe_hrtimer_interrupt, NULL);
  20414. + if (ret) {
  20415. + pr_info("wakeup trace: Couldn't assign "
  20416. + "probe_hrtimer_interrupt "
  20417. + "to trace_hrtimer_interrupt\n");
  20418. + return ret;
  20419. + }
  20420. + break;
  20421. +#endif
  20422. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  20423. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20424. + case TIMERANDWAKEUP_LATENCY:
  20425. + if (!wakeup_latency_enabled_data.enabled ||
  20426. + !missed_timer_offsets_enabled_data.enabled)
  20427. + return -EINVAL;
  20428. + break;
  20429. +#endif
  20430. + default:
  20431. + break;
  20432. + }
  20433. + } else {
  20434. + switch (ed->latency_type) {
  20435. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  20436. + case PREEMPTIRQSOFF_LATENCY:
  20437. + {
  20438. + int cpu;
  20439. +
  20440. + unregister_trace_preemptirqsoff_hist(
  20441. + probe_preemptirqsoff_hist, NULL);
  20442. + for_each_online_cpu(cpu) {
  20443. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  20444. + per_cpu(hist_irqsoff_counting,
  20445. + cpu) = 0;
  20446. +#endif
  20447. +#ifdef CONFIG_PREEMPT_OFF_HIST
  20448. + per_cpu(hist_preemptoff_counting,
  20449. + cpu) = 0;
  20450. +#endif
  20451. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  20452. + per_cpu(hist_preemptirqsoff_counting,
  20453. + cpu) = 0;
  20454. +#endif
  20455. + }
  20456. + }
  20457. + break;
  20458. +#endif
  20459. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20460. + case WAKEUP_LATENCY:
  20461. + {
  20462. + int cpu;
  20463. +
  20464. + unregister_trace_sched_wakeup(
  20465. + probe_wakeup_latency_hist_start, NULL);
  20466. + unregister_trace_sched_wakeup_new(
  20467. + probe_wakeup_latency_hist_start, NULL);
  20468. + unregister_trace_sched_switch(
  20469. + probe_wakeup_latency_hist_stop, NULL);
  20470. + unregister_trace_sched_migrate_task(
  20471. + probe_sched_migrate_task, NULL);
  20472. +
  20473. + for_each_online_cpu(cpu) {
  20474. + per_cpu(wakeup_task, cpu) = NULL;
  20475. + per_cpu(wakeup_sharedprio, cpu) = 0;
  20476. + }
  20477. + }
  20478. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20479. + timerandwakeup_enabled_data.enabled = 0;
  20480. +#endif
  20481. + break;
  20482. +#endif
  20483. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20484. + case MISSED_TIMER_OFFSETS:
  20485. + unregister_trace_hrtimer_interrupt(
  20486. + probe_hrtimer_interrupt, NULL);
  20487. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20488. + timerandwakeup_enabled_data.enabled = 0;
  20489. +#endif
  20490. + break;
  20491. +#endif
  20492. + default:
  20493. + break;
  20494. + }
  20495. + }
  20496. + ed->enabled = enable;
  20497. + return cnt;
  20498. +}
  20499. +
  20500. +static const struct file_operations latency_hist_reset_fops = {
  20501. + .open = tracing_open_generic,
  20502. + .write = latency_hist_reset,
  20503. +};
  20504. +
  20505. +static const struct file_operations enable_fops = {
  20506. + .open = tracing_open_generic,
  20507. + .read = show_enable,
  20508. + .write = do_enable,
  20509. +};
  20510. +
  20511. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20512. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20513. +static const struct file_operations pid_fops = {
  20514. + .open = tracing_open_generic,
  20515. + .read = show_pid,
  20516. + .write = do_pid,
  20517. +};
  20518. +
  20519. +static const struct file_operations maxlatproc_fops = {
  20520. + .open = tracing_open_generic,
  20521. + .read = show_maxlatproc,
  20522. +};
  20523. +#endif
  20524. +
  20525. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  20526. +static notrace void probe_preemptirqsoff_hist(void *v, int reason,
  20527. + int starthist)
  20528. +{
  20529. + int cpu = raw_smp_processor_id();
  20530. + int time_set = 0;
  20531. +
  20532. + if (starthist) {
  20533. + cycle_t uninitialized_var(start);
  20534. +
  20535. + if (!preempt_count() && !irqs_disabled())
  20536. + return;
  20537. +
  20538. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  20539. + if ((reason == IRQS_OFF || reason == TRACE_START) &&
  20540. + !per_cpu(hist_irqsoff_counting, cpu)) {
  20541. + per_cpu(hist_irqsoff_counting, cpu) = 1;
  20542. + start = ftrace_now(cpu);
  20543. + time_set++;
  20544. + per_cpu(hist_irqsoff_start, cpu) = start;
  20545. + }
  20546. +#endif
  20547. +
  20548. +#ifdef CONFIG_PREEMPT_OFF_HIST
  20549. + if ((reason == PREEMPT_OFF || reason == TRACE_START) &&
  20550. + !per_cpu(hist_preemptoff_counting, cpu)) {
  20551. + per_cpu(hist_preemptoff_counting, cpu) = 1;
  20552. + if (!(time_set++))
  20553. + start = ftrace_now(cpu);
  20554. + per_cpu(hist_preemptoff_start, cpu) = start;
  20555. + }
  20556. +#endif
  20557. +
  20558. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  20559. + if (per_cpu(hist_irqsoff_counting, cpu) &&
  20560. + per_cpu(hist_preemptoff_counting, cpu) &&
  20561. + !per_cpu(hist_preemptirqsoff_counting, cpu)) {
  20562. + per_cpu(hist_preemptirqsoff_counting, cpu) = 1;
  20563. + if (!time_set)
  20564. + start = ftrace_now(cpu);
  20565. + per_cpu(hist_preemptirqsoff_start, cpu) = start;
  20566. + }
  20567. +#endif
  20568. + } else {
  20569. + cycle_t uninitialized_var(stop);
  20570. +
  20571. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  20572. + if ((reason == IRQS_ON || reason == TRACE_STOP) &&
  20573. + per_cpu(hist_irqsoff_counting, cpu)) {
  20574. + cycle_t start = per_cpu(hist_irqsoff_start, cpu);
  20575. +
  20576. + stop = ftrace_now(cpu);
  20577. + time_set++;
  20578. + if (start) {
  20579. + long latency = ((long) (stop - start)) /
  20580. + NSECS_PER_USECS;
  20581. +
  20582. + latency_hist(IRQSOFF_LATENCY, cpu, latency, 0,
  20583. + stop, NULL);
  20584. + }
  20585. + per_cpu(hist_irqsoff_counting, cpu) = 0;
  20586. + }
  20587. +#endif
  20588. +
  20589. +#ifdef CONFIG_PREEMPT_OFF_HIST
  20590. + if ((reason == PREEMPT_ON || reason == TRACE_STOP) &&
  20591. + per_cpu(hist_preemptoff_counting, cpu)) {
  20592. + cycle_t start = per_cpu(hist_preemptoff_start, cpu);
  20593. +
  20594. + if (!(time_set++))
  20595. + stop = ftrace_now(cpu);
  20596. + if (start) {
  20597. + long latency = ((long) (stop - start)) /
  20598. + NSECS_PER_USECS;
  20599. +
  20600. + latency_hist(PREEMPTOFF_LATENCY, cpu, latency,
  20601. + 0, stop, NULL);
  20602. + }
  20603. + per_cpu(hist_preemptoff_counting, cpu) = 0;
  20604. + }
  20605. +#endif
  20606. +
  20607. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  20608. + if ((!per_cpu(hist_irqsoff_counting, cpu) ||
  20609. + !per_cpu(hist_preemptoff_counting, cpu)) &&
  20610. + per_cpu(hist_preemptirqsoff_counting, cpu)) {
  20611. + cycle_t start = per_cpu(hist_preemptirqsoff_start, cpu);
  20612. +
  20613. + if (!time_set)
  20614. + stop = ftrace_now(cpu);
  20615. + if (start) {
  20616. + long latency = ((long) (stop - start)) /
  20617. + NSECS_PER_USECS;
  20618. +
  20619. + latency_hist(PREEMPTIRQSOFF_LATENCY, cpu,
  20620. + latency, 0, stop, NULL);
  20621. + }
  20622. + per_cpu(hist_preemptirqsoff_counting, cpu) = 0;
  20623. + }
  20624. +#endif
  20625. + }
  20626. +}
  20627. +#endif
  20628. +
  20629. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20630. +static DEFINE_RAW_SPINLOCK(wakeup_lock);
  20631. +static notrace void probe_sched_migrate_task(void *v, struct task_struct *task,
  20632. + int cpu)
  20633. +{
  20634. + int old_cpu = task_cpu(task);
  20635. +
  20636. + if (cpu != old_cpu) {
  20637. + unsigned long flags;
  20638. + struct task_struct *cpu_wakeup_task;
  20639. +
  20640. + raw_spin_lock_irqsave(&wakeup_lock, flags);
  20641. +
  20642. + cpu_wakeup_task = per_cpu(wakeup_task, old_cpu);
  20643. + if (task == cpu_wakeup_task) {
  20644. + put_task_struct(cpu_wakeup_task);
  20645. + per_cpu(wakeup_task, old_cpu) = NULL;
  20646. + cpu_wakeup_task = per_cpu(wakeup_task, cpu) = task;
  20647. + get_task_struct(cpu_wakeup_task);
  20648. + }
  20649. +
  20650. + raw_spin_unlock_irqrestore(&wakeup_lock, flags);
  20651. + }
  20652. +}
  20653. +
  20654. +static notrace void probe_wakeup_latency_hist_start(void *v,
  20655. + struct task_struct *p)
  20656. +{
  20657. + unsigned long flags;
  20658. + struct task_struct *curr = current;
  20659. + int cpu = task_cpu(p);
  20660. + struct task_struct *cpu_wakeup_task;
  20661. +
  20662. + raw_spin_lock_irqsave(&wakeup_lock, flags);
  20663. +
  20664. + cpu_wakeup_task = per_cpu(wakeup_task, cpu);
  20665. +
  20666. + if (wakeup_pid) {
  20667. + if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) ||
  20668. + p->prio == curr->prio)
  20669. + per_cpu(wakeup_sharedprio, cpu) = 1;
  20670. + if (likely(wakeup_pid != task_pid_nr(p)))
  20671. + goto out;
  20672. + } else {
  20673. + if (likely(!rt_task(p)) ||
  20674. + (cpu_wakeup_task && p->prio > cpu_wakeup_task->prio) ||
  20675. + p->prio > curr->prio)
  20676. + goto out;
  20677. + if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) ||
  20678. + p->prio == curr->prio)
  20679. + per_cpu(wakeup_sharedprio, cpu) = 1;
  20680. + }
  20681. +
  20682. + if (cpu_wakeup_task)
  20683. + put_task_struct(cpu_wakeup_task);
  20684. + cpu_wakeup_task = per_cpu(wakeup_task, cpu) = p;
  20685. + get_task_struct(cpu_wakeup_task);
  20686. + cpu_wakeup_task->preempt_timestamp_hist =
  20687. + ftrace_now(raw_smp_processor_id());
  20688. +out:
  20689. + raw_spin_unlock_irqrestore(&wakeup_lock, flags);
  20690. +}
  20691. +
  20692. +static notrace void probe_wakeup_latency_hist_stop(void *v,
  20693. + bool preempt, struct task_struct *prev, struct task_struct *next)
  20694. +{
  20695. + unsigned long flags;
  20696. + int cpu = task_cpu(next);
  20697. + long latency;
  20698. + cycle_t stop;
  20699. + struct task_struct *cpu_wakeup_task;
  20700. +
  20701. + raw_spin_lock_irqsave(&wakeup_lock, flags);
  20702. +
  20703. + cpu_wakeup_task = per_cpu(wakeup_task, cpu);
  20704. +
  20705. + if (cpu_wakeup_task == NULL)
  20706. + goto out;
  20707. +
  20708. + /* Already running? */
  20709. + if (unlikely(current == cpu_wakeup_task))
  20710. + goto out_reset;
  20711. +
  20712. + if (next != cpu_wakeup_task) {
  20713. + if (next->prio < cpu_wakeup_task->prio)
  20714. + goto out_reset;
  20715. +
  20716. + if (next->prio == cpu_wakeup_task->prio)
  20717. + per_cpu(wakeup_sharedprio, cpu) = 1;
  20718. +
  20719. + goto out;
  20720. + }
  20721. +
  20722. + if (current->prio == cpu_wakeup_task->prio)
  20723. + per_cpu(wakeup_sharedprio, cpu) = 1;
  20724. +
  20725. + /*
  20726. + * The task we are waiting for is about to be switched to.
  20727. + * Calculate latency and store it in histogram.
  20728. + */
  20729. + stop = ftrace_now(raw_smp_processor_id());
  20730. +
  20731. + latency = ((long) (stop - next->preempt_timestamp_hist)) /
  20732. + NSECS_PER_USECS;
  20733. +
  20734. + if (per_cpu(wakeup_sharedprio, cpu)) {
  20735. + latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, 0, stop,
  20736. + next);
  20737. + per_cpu(wakeup_sharedprio, cpu) = 0;
  20738. + } else {
  20739. + latency_hist(WAKEUP_LATENCY, cpu, latency, 0, stop, next);
  20740. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20741. + if (timerandwakeup_enabled_data.enabled) {
  20742. + latency_hist(TIMERANDWAKEUP_LATENCY, cpu,
  20743. + next->timer_offset + latency, next->timer_offset,
  20744. + stop, next);
  20745. + }
  20746. +#endif
  20747. + }
  20748. +
  20749. +out_reset:
  20750. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20751. + next->timer_offset = 0;
  20752. +#endif
  20753. + put_task_struct(cpu_wakeup_task);
  20754. + per_cpu(wakeup_task, cpu) = NULL;
  20755. +out:
  20756. + raw_spin_unlock_irqrestore(&wakeup_lock, flags);
  20757. +}
  20758. +#endif
  20759. +
  20760. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20761. +static notrace void probe_hrtimer_interrupt(void *v, int cpu,
  20762. + long long latency_ns, struct task_struct *curr,
  20763. + struct task_struct *task)
  20764. +{
  20765. + if (latency_ns <= 0 && task != NULL && rt_task(task) &&
  20766. + (task->prio < curr->prio ||
  20767. + (task->prio == curr->prio &&
  20768. + !cpumask_test_cpu(cpu, &task->cpus_allowed)))) {
  20769. + long latency;
  20770. + cycle_t now;
  20771. +
  20772. + if (missed_timer_offsets_pid) {
  20773. + if (likely(missed_timer_offsets_pid !=
  20774. + task_pid_nr(task)))
  20775. + return;
  20776. + }
  20777. +
  20778. + now = ftrace_now(cpu);
  20779. + latency = (long) div_s64(-latency_ns, NSECS_PER_USECS);
  20780. + latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, latency, now,
  20781. + task);
  20782. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20783. + task->timer_offset = latency;
  20784. +#endif
  20785. + }
  20786. +}
  20787. +#endif
  20788. +
  20789. +static __init int latency_hist_init(void)
  20790. +{
  20791. + struct dentry *latency_hist_root = NULL;
  20792. + struct dentry *dentry;
  20793. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20794. + struct dentry *dentry_sharedprio;
  20795. +#endif
  20796. + struct dentry *entry;
  20797. + struct dentry *enable_root;
  20798. + int i = 0;
  20799. + struct hist_data *my_hist;
  20800. + char name[64];
  20801. + char *cpufmt = "CPU%d";
  20802. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  20803. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20804. + char *cpufmt_maxlatproc = "max_latency-CPU%d";
  20805. + struct maxlatproc_data *mp = NULL;
  20806. +#endif
  20807. +
  20808. + dentry = tracing_init_dentry();
  20809. + latency_hist_root = debugfs_create_dir(latency_hist_dir_root, dentry);
  20810. + enable_root = debugfs_create_dir("enable", latency_hist_root);
  20811. +
  20812. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  20813. + dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root);
  20814. + for_each_possible_cpu(i) {
  20815. + sprintf(name, cpufmt, i);
  20816. + entry = debugfs_create_file(name, 0444, dentry,
  20817. + &per_cpu(irqsoff_hist, i), &latency_hist_fops);
  20818. + my_hist = &per_cpu(irqsoff_hist, i);
  20819. + atomic_set(&my_hist->hist_mode, 1);
  20820. + my_hist->min_lat = LONG_MAX;
  20821. + }
  20822. + entry = debugfs_create_file("reset", 0644, dentry,
  20823. + (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops);
  20824. +#endif
  20825. +
  20826. +#ifdef CONFIG_PREEMPT_OFF_HIST
  20827. + dentry = debugfs_create_dir(preemptoff_hist_dir,
  20828. + latency_hist_root);
  20829. + for_each_possible_cpu(i) {
  20830. + sprintf(name, cpufmt, i);
  20831. + entry = debugfs_create_file(name, 0444, dentry,
  20832. + &per_cpu(preemptoff_hist, i), &latency_hist_fops);
  20833. + my_hist = &per_cpu(preemptoff_hist, i);
  20834. + atomic_set(&my_hist->hist_mode, 1);
  20835. + my_hist->min_lat = LONG_MAX;
  20836. + }
  20837. + entry = debugfs_create_file("reset", 0644, dentry,
  20838. + (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops);
  20839. +#endif
  20840. +
  20841. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  20842. + dentry = debugfs_create_dir(preemptirqsoff_hist_dir,
  20843. + latency_hist_root);
  20844. + for_each_possible_cpu(i) {
  20845. + sprintf(name, cpufmt, i);
  20846. + entry = debugfs_create_file(name, 0444, dentry,
  20847. + &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops);
  20848. + my_hist = &per_cpu(preemptirqsoff_hist, i);
  20849. + atomic_set(&my_hist->hist_mode, 1);
  20850. + my_hist->min_lat = LONG_MAX;
  20851. + }
  20852. + entry = debugfs_create_file("reset", 0644, dentry,
  20853. + (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops);
  20854. +#endif
  20855. +
  20856. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  20857. + entry = debugfs_create_file("preemptirqsoff", 0644,
  20858. + enable_root, (void *)&preemptirqsoff_enabled_data,
  20859. + &enable_fops);
  20860. +#endif
  20861. +
  20862. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20863. + dentry = debugfs_create_dir(wakeup_latency_hist_dir,
  20864. + latency_hist_root);
  20865. + dentry_sharedprio = debugfs_create_dir(
  20866. + wakeup_latency_hist_dir_sharedprio, dentry);
  20867. + for_each_possible_cpu(i) {
  20868. + sprintf(name, cpufmt, i);
  20869. +
  20870. + entry = debugfs_create_file(name, 0444, dentry,
  20871. + &per_cpu(wakeup_latency_hist, i),
  20872. + &latency_hist_fops);
  20873. + my_hist = &per_cpu(wakeup_latency_hist, i);
  20874. + atomic_set(&my_hist->hist_mode, 1);
  20875. + my_hist->min_lat = LONG_MAX;
  20876. +
  20877. + entry = debugfs_create_file(name, 0444, dentry_sharedprio,
  20878. + &per_cpu(wakeup_latency_hist_sharedprio, i),
  20879. + &latency_hist_fops);
  20880. + my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i);
  20881. + atomic_set(&my_hist->hist_mode, 1);
  20882. + my_hist->min_lat = LONG_MAX;
  20883. +
  20884. + sprintf(name, cpufmt_maxlatproc, i);
  20885. +
  20886. + mp = &per_cpu(wakeup_maxlatproc, i);
  20887. + entry = debugfs_create_file(name, 0444, dentry, mp,
  20888. + &maxlatproc_fops);
  20889. + clear_maxlatprocdata(mp);
  20890. +
  20891. + mp = &per_cpu(wakeup_maxlatproc_sharedprio, i);
  20892. + entry = debugfs_create_file(name, 0444, dentry_sharedprio, mp,
  20893. + &maxlatproc_fops);
  20894. + clear_maxlatprocdata(mp);
  20895. + }
  20896. + entry = debugfs_create_file("pid", 0644, dentry,
  20897. + (void *)&wakeup_pid, &pid_fops);
  20898. + entry = debugfs_create_file("reset", 0644, dentry,
  20899. + (void *)WAKEUP_LATENCY, &latency_hist_reset_fops);
  20900. + entry = debugfs_create_file("reset", 0644, dentry_sharedprio,
  20901. + (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops);
  20902. + entry = debugfs_create_file("wakeup", 0644,
  20903. + enable_root, (void *)&wakeup_latency_enabled_data,
  20904. + &enable_fops);
  20905. +#endif
  20906. +
  20907. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20908. + dentry = debugfs_create_dir(missed_timer_offsets_dir,
  20909. + latency_hist_root);
  20910. + for_each_possible_cpu(i) {
  20911. + sprintf(name, cpufmt, i);
  20912. + entry = debugfs_create_file(name, 0444, dentry,
  20913. + &per_cpu(missed_timer_offsets, i), &latency_hist_fops);
  20914. + my_hist = &per_cpu(missed_timer_offsets, i);
  20915. + atomic_set(&my_hist->hist_mode, 1);
  20916. + my_hist->min_lat = LONG_MAX;
  20917. +
  20918. + sprintf(name, cpufmt_maxlatproc, i);
  20919. + mp = &per_cpu(missed_timer_offsets_maxlatproc, i);
  20920. + entry = debugfs_create_file(name, 0444, dentry, mp,
  20921. + &maxlatproc_fops);
  20922. + clear_maxlatprocdata(mp);
  20923. + }
  20924. + entry = debugfs_create_file("pid", 0644, dentry,
  20925. + (void *)&missed_timer_offsets_pid, &pid_fops);
  20926. + entry = debugfs_create_file("reset", 0644, dentry,
  20927. + (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops);
  20928. + entry = debugfs_create_file("missed_timer_offsets", 0644,
  20929. + enable_root, (void *)&missed_timer_offsets_enabled_data,
  20930. + &enable_fops);
  20931. +#endif
  20932. +
  20933. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  20934. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20935. + dentry = debugfs_create_dir(timerandwakeup_latency_hist_dir,
  20936. + latency_hist_root);
  20937. + for_each_possible_cpu(i) {
  20938. + sprintf(name, cpufmt, i);
  20939. + entry = debugfs_create_file(name, 0444, dentry,
  20940. + &per_cpu(timerandwakeup_latency_hist, i),
  20941. + &latency_hist_fops);
  20942. + my_hist = &per_cpu(timerandwakeup_latency_hist, i);
  20943. + atomic_set(&my_hist->hist_mode, 1);
  20944. + my_hist->min_lat = LONG_MAX;
  20945. +
  20946. + sprintf(name, cpufmt_maxlatproc, i);
  20947. + mp = &per_cpu(timerandwakeup_maxlatproc, i);
  20948. + entry = debugfs_create_file(name, 0444, dentry, mp,
  20949. + &maxlatproc_fops);
  20950. + clear_maxlatprocdata(mp);
  20951. + }
  20952. + entry = debugfs_create_file("reset", 0644, dentry,
  20953. + (void *)TIMERANDWAKEUP_LATENCY, &latency_hist_reset_fops);
  20954. + entry = debugfs_create_file("timerandwakeup", 0644,
  20955. + enable_root, (void *)&timerandwakeup_enabled_data,
  20956. + &enable_fops);
  20957. +#endif
  20958. + return 0;
  20959. +}
  20960. +
  20961. +device_initcall(latency_hist_init);
  20962. diff -Nur linux-4.6.6.orig/kernel/trace/Makefile linux-4.6.6/kernel/trace/Makefile
  20963. --- linux-4.6.6.orig/kernel/trace/Makefile 2016-08-10 12:54:57.000000000 +0200
  20964. +++ linux-4.6.6/kernel/trace/Makefile 2016-08-22 21:27:47.425893881 +0200
  20965. @@ -36,6 +36,10 @@
  20966. obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
  20967. obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o
  20968. obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o
  20969. +obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o
  20970. +obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o
  20971. +obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o
  20972. +obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o
  20973. obj-$(CONFIG_NOP_TRACER) += trace_nop.o
  20974. obj-$(CONFIG_STACK_TRACER) += trace_stack.o
  20975. obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
  20976. diff -Nur linux-4.6.6.orig/kernel/trace/trace.c linux-4.6.6/kernel/trace/trace.c
  20977. --- linux-4.6.6.orig/kernel/trace/trace.c 2016-08-10 12:54:57.000000000 +0200
  20978. +++ linux-4.6.6/kernel/trace/trace.c 2016-08-22 21:27:47.429894141 +0200
  20979. @@ -1657,6 +1657,7 @@
  20980. struct task_struct *tsk = current;
  20981. entry->preempt_count = pc & 0xff;
  20982. + entry->preempt_lazy_count = preempt_lazy_count();
  20983. entry->pid = (tsk) ? tsk->pid : 0;
  20984. entry->flags =
  20985. #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
  20986. @@ -1667,8 +1668,11 @@
  20987. ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
  20988. ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
  20989. ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
  20990. - (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
  20991. + (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) |
  20992. + (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) |
  20993. (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
  20994. +
  20995. + entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0;
  20996. }
  20997. EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
  20998. @@ -2561,14 +2565,17 @@
  20999. static void print_lat_help_header(struct seq_file *m)
  21000. {
  21001. - seq_puts(m, "# _------=> CPU# \n"
  21002. - "# / _-----=> irqs-off \n"
  21003. - "# | / _----=> need-resched \n"
  21004. - "# || / _---=> hardirq/softirq \n"
  21005. - "# ||| / _--=> preempt-depth \n"
  21006. - "# |||| / delay \n"
  21007. - "# cmd pid ||||| time | caller \n"
  21008. - "# \\ / ||||| \\ | / \n");
  21009. + seq_puts(m, "# _--------=> CPU# \n"
  21010. + "# / _-------=> irqs-off \n"
  21011. + "# | / _------=> need-resched \n"
  21012. + "# || / _-----=> need-resched_lazy \n"
  21013. + "# ||| / _----=> hardirq/softirq \n"
  21014. + "# |||| / _---=> preempt-depth \n"
  21015. + "# ||||| / _--=> preempt-lazy-depth\n"
  21016. + "# |||||| / _-=> migrate-disable \n"
  21017. + "# ||||||| / delay \n"
  21018. + "# cmd pid |||||||| time | caller \n"
  21019. + "# \\ / |||||||| \\ | / \n");
  21020. }
  21021. static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
  21022. @@ -2594,11 +2601,14 @@
  21023. print_event_info(buf, m);
  21024. seq_puts(m, "# _-----=> irqs-off\n"
  21025. "# / _----=> need-resched\n"
  21026. - "# | / _---=> hardirq/softirq\n"
  21027. - "# || / _--=> preempt-depth\n"
  21028. - "# ||| / delay\n"
  21029. - "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
  21030. - "# | | | |||| | |\n");
  21031. + "# |/ _-----=> need-resched_lazy\n"
  21032. + "# || / _---=> hardirq/softirq\n"
  21033. + "# ||| / _--=> preempt-depth\n"
  21034. + "# |||| /_--=> preempt-lazy-depth\n"
  21035. + "# ||||| _-=> migrate-disable \n"
  21036. + "# ||||| / delay\n"
  21037. + "# TASK-PID CPU# |||||| TIMESTAMP FUNCTION\n"
  21038. + "# | | | |||||| | |\n");
  21039. }
  21040. void
  21041. diff -Nur linux-4.6.6.orig/kernel/trace/trace_events.c linux-4.6.6/kernel/trace/trace_events.c
  21042. --- linux-4.6.6.orig/kernel/trace/trace_events.c 2016-08-10 12:54:57.000000000 +0200
  21043. +++ linux-4.6.6/kernel/trace/trace_events.c 2016-08-22 21:27:47.429894141 +0200
  21044. @@ -188,6 +188,8 @@
  21045. __common_field(unsigned char, flags);
  21046. __common_field(unsigned char, preempt_count);
  21047. __common_field(int, pid);
  21048. + __common_field(unsigned short, migrate_disable);
  21049. + __common_field(unsigned short, padding);
  21050. return ret;
  21051. }
  21052. @@ -244,6 +246,14 @@
  21053. local_save_flags(fbuffer->flags);
  21054. fbuffer->pc = preempt_count();
  21055. + /*
  21056. + * If CONFIG_PREEMPT is enabled, then the tracepoint itself disables
  21057. + * preemption (adding one to the preempt_count). Since we are
  21058. + * interested in the preempt_count at the time the tracepoint was
  21059. + * hit, we need to subtract one to offset the increment.
  21060. + */
  21061. + if (IS_ENABLED(CONFIG_PREEMPT))
  21062. + fbuffer->pc--;
  21063. fbuffer->trace_file = trace_file;
  21064. fbuffer->event =
  21065. diff -Nur linux-4.6.6.orig/kernel/trace/trace.h linux-4.6.6/kernel/trace/trace.h
  21066. --- linux-4.6.6.orig/kernel/trace/trace.h 2016-08-10 12:54:57.000000000 +0200
  21067. +++ linux-4.6.6/kernel/trace/trace.h 2016-08-22 21:27:47.429894141 +0200
  21068. @@ -117,6 +117,7 @@
  21069. * NEED_RESCHED - reschedule is requested
  21070. * HARDIRQ - inside an interrupt handler
  21071. * SOFTIRQ - inside a softirq handler
  21072. + * NEED_RESCHED_LAZY - lazy reschedule is requested
  21073. */
  21074. enum trace_flag_type {
  21075. TRACE_FLAG_IRQS_OFF = 0x01,
  21076. @@ -126,6 +127,7 @@
  21077. TRACE_FLAG_SOFTIRQ = 0x10,
  21078. TRACE_FLAG_PREEMPT_RESCHED = 0x20,
  21079. TRACE_FLAG_NMI = 0x40,
  21080. + TRACE_FLAG_NEED_RESCHED_LAZY = 0x80,
  21081. };
  21082. #define TRACE_BUF_SIZE 1024
  21083. diff -Nur linux-4.6.6.orig/kernel/trace/trace_irqsoff.c linux-4.6.6/kernel/trace/trace_irqsoff.c
  21084. --- linux-4.6.6.orig/kernel/trace/trace_irqsoff.c 2016-08-10 12:54:57.000000000 +0200
  21085. +++ linux-4.6.6/kernel/trace/trace_irqsoff.c 2016-08-22 21:27:47.429894141 +0200
  21086. @@ -13,6 +13,7 @@
  21087. #include <linux/uaccess.h>
  21088. #include <linux/module.h>
  21089. #include <linux/ftrace.h>
  21090. +#include <trace/events/hist.h>
  21091. #include "trace.h"
  21092. @@ -424,11 +425,13 @@
  21093. {
  21094. if (preempt_trace() || irq_trace())
  21095. start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  21096. + trace_preemptirqsoff_hist_rcuidle(TRACE_START, 1);
  21097. }
  21098. EXPORT_SYMBOL_GPL(start_critical_timings);
  21099. void stop_critical_timings(void)
  21100. {
  21101. + trace_preemptirqsoff_hist_rcuidle(TRACE_STOP, 0);
  21102. if (preempt_trace() || irq_trace())
  21103. stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  21104. }
  21105. @@ -438,6 +441,7 @@
  21106. #ifdef CONFIG_PROVE_LOCKING
  21107. void time_hardirqs_on(unsigned long a0, unsigned long a1)
  21108. {
  21109. + trace_preemptirqsoff_hist_rcuidle(IRQS_ON, 0);
  21110. if (!preempt_trace() && irq_trace())
  21111. stop_critical_timing(a0, a1);
  21112. }
  21113. @@ -446,6 +450,7 @@
  21114. {
  21115. if (!preempt_trace() && irq_trace())
  21116. start_critical_timing(a0, a1);
  21117. + trace_preemptirqsoff_hist_rcuidle(IRQS_OFF, 1);
  21118. }
  21119. #else /* !CONFIG_PROVE_LOCKING */
  21120. @@ -471,6 +476,7 @@
  21121. */
  21122. void trace_hardirqs_on(void)
  21123. {
  21124. + trace_preemptirqsoff_hist(IRQS_ON, 0);
  21125. if (!preempt_trace() && irq_trace())
  21126. stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  21127. }
  21128. @@ -480,11 +486,13 @@
  21129. {
  21130. if (!preempt_trace() && irq_trace())
  21131. start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  21132. + trace_preemptirqsoff_hist(IRQS_OFF, 1);
  21133. }
  21134. EXPORT_SYMBOL(trace_hardirqs_off);
  21135. __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
  21136. {
  21137. + trace_preemptirqsoff_hist(IRQS_ON, 0);
  21138. if (!preempt_trace() && irq_trace())
  21139. stop_critical_timing(CALLER_ADDR0, caller_addr);
  21140. }
  21141. @@ -494,6 +502,7 @@
  21142. {
  21143. if (!preempt_trace() && irq_trace())
  21144. start_critical_timing(CALLER_ADDR0, caller_addr);
  21145. + trace_preemptirqsoff_hist(IRQS_OFF, 1);
  21146. }
  21147. EXPORT_SYMBOL(trace_hardirqs_off_caller);
  21148. @@ -503,12 +512,14 @@
  21149. #ifdef CONFIG_PREEMPT_TRACER
  21150. void trace_preempt_on(unsigned long a0, unsigned long a1)
  21151. {
  21152. + trace_preemptirqsoff_hist(PREEMPT_ON, 0);
  21153. if (preempt_trace() && !irq_trace())
  21154. stop_critical_timing(a0, a1);
  21155. }
  21156. void trace_preempt_off(unsigned long a0, unsigned long a1)
  21157. {
  21158. + trace_preemptirqsoff_hist(PREEMPT_ON, 1);
  21159. if (preempt_trace() && !irq_trace())
  21160. start_critical_timing(a0, a1);
  21161. }
  21162. diff -Nur linux-4.6.6.orig/kernel/trace/trace_output.c linux-4.6.6/kernel/trace/trace_output.c
  21163. --- linux-4.6.6.orig/kernel/trace/trace_output.c 2016-08-10 12:54:57.000000000 +0200
  21164. +++ linux-4.6.6/kernel/trace/trace_output.c 2016-08-22 21:27:47.429894141 +0200
  21165. @@ -386,6 +386,7 @@
  21166. {
  21167. char hardsoft_irq;
  21168. char need_resched;
  21169. + char need_resched_lazy;
  21170. char irqs_off;
  21171. int hardirq;
  21172. int softirq;
  21173. @@ -416,6 +417,9 @@
  21174. break;
  21175. }
  21176. + need_resched_lazy =
  21177. + (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.';
  21178. +
  21179. hardsoft_irq =
  21180. (nmi && hardirq) ? 'Z' :
  21181. nmi ? 'z' :
  21182. @@ -424,14 +428,25 @@
  21183. softirq ? 's' :
  21184. '.' ;
  21185. - trace_seq_printf(s, "%c%c%c",
  21186. - irqs_off, need_resched, hardsoft_irq);
  21187. + trace_seq_printf(s, "%c%c%c%c",
  21188. + irqs_off, need_resched, need_resched_lazy,
  21189. + hardsoft_irq);
  21190. if (entry->preempt_count)
  21191. trace_seq_printf(s, "%x", entry->preempt_count);
  21192. else
  21193. trace_seq_putc(s, '.');
  21194. + if (entry->preempt_lazy_count)
  21195. + trace_seq_printf(s, "%x", entry->preempt_lazy_count);
  21196. + else
  21197. + trace_seq_putc(s, '.');
  21198. +
  21199. + if (entry->migrate_disable)
  21200. + trace_seq_printf(s, "%x", entry->migrate_disable);
  21201. + else
  21202. + trace_seq_putc(s, '.');
  21203. +
  21204. return !trace_seq_has_overflowed(s);
  21205. }
  21206. diff -Nur linux-4.6.6.orig/kernel/user.c linux-4.6.6/kernel/user.c
  21207. --- linux-4.6.6.orig/kernel/user.c 2016-08-10 12:54:57.000000000 +0200
  21208. +++ linux-4.6.6/kernel/user.c 2016-08-22 21:27:47.429894141 +0200
  21209. @@ -161,11 +161,11 @@
  21210. if (!up)
  21211. return;
  21212. - local_irq_save(flags);
  21213. + local_irq_save_nort(flags);
  21214. if (atomic_dec_and_lock(&up->__count, &uidhash_lock))
  21215. free_user(up, flags);
  21216. else
  21217. - local_irq_restore(flags);
  21218. + local_irq_restore_nort(flags);
  21219. }
  21220. struct user_struct *alloc_uid(kuid_t uid)
  21221. diff -Nur linux-4.6.6.orig/kernel/watchdog.c linux-4.6.6/kernel/watchdog.c
  21222. --- linux-4.6.6.orig/kernel/watchdog.c 2016-08-10 12:54:57.000000000 +0200
  21223. +++ linux-4.6.6/kernel/watchdog.c 2016-08-22 21:27:47.429894141 +0200
  21224. @@ -315,6 +315,8 @@
  21225. #ifdef CONFIG_HARDLOCKUP_DETECTOR
  21226. +static DEFINE_RAW_SPINLOCK(watchdog_output_lock);
  21227. +
  21228. static struct perf_event_attr wd_hw_attr = {
  21229. .type = PERF_TYPE_HARDWARE,
  21230. .config = PERF_COUNT_HW_CPU_CYCLES,
  21231. @@ -349,6 +351,13 @@
  21232. /* only print hardlockups once */
  21233. if (__this_cpu_read(hard_watchdog_warn) == true)
  21234. return;
  21235. + /*
  21236. + * If early-printk is enabled then make sure we do not
  21237. + * lock up in printk() and kill console logging:
  21238. + */
  21239. + printk_kill();
  21240. +
  21241. + raw_spin_lock(&watchdog_output_lock);
  21242. pr_emerg("Watchdog detected hard LOCKUP on cpu %d", this_cpu);
  21243. print_modules();
  21244. @@ -366,6 +375,7 @@
  21245. !test_and_set_bit(0, &hardlockup_allcpu_dumped))
  21246. trigger_allbutself_cpu_backtrace();
  21247. + raw_spin_unlock(&watchdog_output_lock);
  21248. if (hardlockup_panic)
  21249. nmi_panic(regs, "Hard LOCKUP");
  21250. @@ -513,6 +523,7 @@
  21251. /* kick off the timer for the hardlockup detector */
  21252. hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  21253. hrtimer->function = watchdog_timer_fn;
  21254. + hrtimer->irqsafe = 1;
  21255. /* Enable the perf event */
  21256. watchdog_nmi_enable(cpu);
  21257. diff -Nur linux-4.6.6.orig/kernel/workqueue.c linux-4.6.6/kernel/workqueue.c
  21258. --- linux-4.6.6.orig/kernel/workqueue.c 2016-08-10 12:54:57.000000000 +0200
  21259. +++ linux-4.6.6/kernel/workqueue.c 2016-08-22 21:27:47.433894394 +0200
  21260. @@ -48,6 +48,8 @@
  21261. #include <linux/nodemask.h>
  21262. #include <linux/moduleparam.h>
  21263. #include <linux/uaccess.h>
  21264. +#include <linux/locallock.h>
  21265. +#include <linux/delay.h>
  21266. #include "workqueue_internal.h"
  21267. @@ -121,11 +123,16 @@
  21268. * cpu or grabbing pool->lock is enough for read access. If
  21269. * POOL_DISASSOCIATED is set, it's identical to L.
  21270. *
  21271. + * On RT we need the extra protection via rt_lock_idle_list() for
  21272. + * the list manipulations against read access from
  21273. + * wq_worker_sleeping(). All other places are nicely serialized via
  21274. + * pool->lock.
  21275. + *
  21276. * A: pool->attach_mutex protected.
  21277. *
  21278. * PL: wq_pool_mutex protected.
  21279. *
  21280. - * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
  21281. + * PR: wq_pool_mutex protected for writes. RCU protected for reads.
  21282. *
  21283. * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
  21284. *
  21285. @@ -134,7 +141,7 @@
  21286. *
  21287. * WQ: wq->mutex protected.
  21288. *
  21289. - * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
  21290. + * WR: wq->mutex protected for writes. RCU protected for reads.
  21291. *
  21292. * MD: wq_mayday_lock protected.
  21293. */
  21294. @@ -185,7 +192,7 @@
  21295. atomic_t nr_running ____cacheline_aligned_in_smp;
  21296. /*
  21297. - * Destruction of pool is sched-RCU protected to allow dereferences
  21298. + * Destruction of pool is RCU protected to allow dereferences
  21299. * from get_work_pool().
  21300. */
  21301. struct rcu_head rcu;
  21302. @@ -214,7 +221,7 @@
  21303. /*
  21304. * Release of unbound pwq is punted to system_wq. See put_pwq()
  21305. * and pwq_unbound_release_workfn() for details. pool_workqueue
  21306. - * itself is also sched-RCU protected so that the first pwq can be
  21307. + * itself is also RCU protected so that the first pwq can be
  21308. * determined without grabbing wq->mutex.
  21309. */
  21310. struct work_struct unbound_release_work;
  21311. @@ -348,6 +355,8 @@
  21312. struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
  21313. EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
  21314. +static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock);
  21315. +
  21316. static int worker_thread(void *__worker);
  21317. static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
  21318. @@ -355,20 +364,20 @@
  21319. #include <trace/events/workqueue.h>
  21320. #define assert_rcu_or_pool_mutex() \
  21321. - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
  21322. + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  21323. !lockdep_is_held(&wq_pool_mutex), \
  21324. - "sched RCU or wq_pool_mutex should be held")
  21325. + "RCU or wq_pool_mutex should be held")
  21326. #define assert_rcu_or_wq_mutex(wq) \
  21327. - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
  21328. + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  21329. !lockdep_is_held(&wq->mutex), \
  21330. - "sched RCU or wq->mutex should be held")
  21331. + "RCU or wq->mutex should be held")
  21332. #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
  21333. - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
  21334. + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  21335. !lockdep_is_held(&wq->mutex) && \
  21336. !lockdep_is_held(&wq_pool_mutex), \
  21337. - "sched RCU, wq->mutex or wq_pool_mutex should be held")
  21338. + "RCU, wq->mutex or wq_pool_mutex should be held")
  21339. #define for_each_cpu_worker_pool(pool, cpu) \
  21340. for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
  21341. @@ -380,7 +389,7 @@
  21342. * @pool: iteration cursor
  21343. * @pi: integer used for iteration
  21344. *
  21345. - * This must be called either with wq_pool_mutex held or sched RCU read
  21346. + * This must be called either with wq_pool_mutex held or RCU read
  21347. * locked. If the pool needs to be used beyond the locking in effect, the
  21348. * caller is responsible for guaranteeing that the pool stays online.
  21349. *
  21350. @@ -412,7 +421,7 @@
  21351. * @pwq: iteration cursor
  21352. * @wq: the target workqueue
  21353. *
  21354. - * This must be called either with wq->mutex held or sched RCU read locked.
  21355. + * This must be called either with wq->mutex held or RCU read locked.
  21356. * If the pwq needs to be used beyond the locking in effect, the caller is
  21357. * responsible for guaranteeing that the pwq stays online.
  21358. *
  21359. @@ -424,6 +433,31 @@
  21360. if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
  21361. else
  21362. +#ifdef CONFIG_PREEMPT_RT_BASE
  21363. +static inline void rt_lock_idle_list(struct worker_pool *pool)
  21364. +{
  21365. + preempt_disable();
  21366. +}
  21367. +static inline void rt_unlock_idle_list(struct worker_pool *pool)
  21368. +{
  21369. + preempt_enable();
  21370. +}
  21371. +static inline void sched_lock_idle_list(struct worker_pool *pool) { }
  21372. +static inline void sched_unlock_idle_list(struct worker_pool *pool) { }
  21373. +#else
  21374. +static inline void rt_lock_idle_list(struct worker_pool *pool) { }
  21375. +static inline void rt_unlock_idle_list(struct worker_pool *pool) { }
  21376. +static inline void sched_lock_idle_list(struct worker_pool *pool)
  21377. +{
  21378. + spin_lock_irq(&pool->lock);
  21379. +}
  21380. +static inline void sched_unlock_idle_list(struct worker_pool *pool)
  21381. +{
  21382. + spin_unlock_irq(&pool->lock);
  21383. +}
  21384. +#endif
  21385. +
  21386. +
  21387. #ifdef CONFIG_DEBUG_OBJECTS_WORK
  21388. static struct debug_obj_descr work_debug_descr;
  21389. @@ -574,7 +608,7 @@
  21390. * @wq: the target workqueue
  21391. * @node: the node ID
  21392. *
  21393. - * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
  21394. + * This must be called with any of wq_pool_mutex, wq->mutex or RCU
  21395. * read locked.
  21396. * If the pwq needs to be used beyond the locking in effect, the caller is
  21397. * responsible for guaranteeing that the pwq stays online.
  21398. @@ -718,8 +752,8 @@
  21399. * @work: the work item of interest
  21400. *
  21401. * Pools are created and destroyed under wq_pool_mutex, and allows read
  21402. - * access under sched-RCU read lock. As such, this function should be
  21403. - * called under wq_pool_mutex or with preemption disabled.
  21404. + * access under RCU read lock. As such, this function should be
  21405. + * called under wq_pool_mutex or inside of a rcu_read_lock() region.
  21406. *
  21407. * All fields of the returned pool are accessible as long as the above
  21408. * mentioned locking is in effect. If the returned pool needs to be used
  21409. @@ -856,50 +890,45 @@
  21410. */
  21411. static void wake_up_worker(struct worker_pool *pool)
  21412. {
  21413. - struct worker *worker = first_idle_worker(pool);
  21414. + struct worker *worker;
  21415. +
  21416. + rt_lock_idle_list(pool);
  21417. +
  21418. + worker = first_idle_worker(pool);
  21419. if (likely(worker))
  21420. wake_up_process(worker->task);
  21421. +
  21422. + rt_unlock_idle_list(pool);
  21423. }
  21424. /**
  21425. - * wq_worker_waking_up - a worker is waking up
  21426. - * @task: task waking up
  21427. + * wq_worker_running - a worker is running again
  21428. * @cpu: CPU @task is waking up to
  21429. *
  21430. - * This function is called during try_to_wake_up() when a worker is
  21431. - * being awoken.
  21432. - *
  21433. - * CONTEXT:
  21434. - * spin_lock_irq(rq->lock)
  21435. + * This function is called when a worker returns from schedule()
  21436. */
  21437. -void wq_worker_waking_up(struct task_struct *task, int cpu)
  21438. +void wq_worker_running(struct task_struct *task)
  21439. {
  21440. struct worker *worker = kthread_data(task);
  21441. - if (!(worker->flags & WORKER_NOT_RUNNING)) {
  21442. - WARN_ON_ONCE(worker->pool->cpu != cpu);
  21443. + if (!worker->sleeping)
  21444. + return;
  21445. + if (!(worker->flags & WORKER_NOT_RUNNING))
  21446. atomic_inc(&worker->pool->nr_running);
  21447. - }
  21448. + worker->sleeping = 0;
  21449. }
  21450. /**
  21451. * wq_worker_sleeping - a worker is going to sleep
  21452. * @task: task going to sleep
  21453. *
  21454. - * This function is called during schedule() when a busy worker is
  21455. - * going to sleep. Worker on the same cpu can be woken up by
  21456. - * returning pointer to its task.
  21457. - *
  21458. - * CONTEXT:
  21459. - * spin_lock_irq(rq->lock)
  21460. - *
  21461. - * Return:
  21462. - * Worker task on @cpu to wake up, %NULL if none.
  21463. + * This function is called from schedule() when a busy worker is
  21464. + * going to sleep.
  21465. */
  21466. -struct task_struct *wq_worker_sleeping(struct task_struct *task)
  21467. +void wq_worker_sleeping(struct task_struct *task)
  21468. {
  21469. - struct worker *worker = kthread_data(task), *to_wakeup = NULL;
  21470. + struct worker *worker = kthread_data(task);
  21471. struct worker_pool *pool;
  21472. /*
  21473. @@ -908,29 +937,26 @@
  21474. * checking NOT_RUNNING.
  21475. */
  21476. if (worker->flags & WORKER_NOT_RUNNING)
  21477. - return NULL;
  21478. + return;
  21479. pool = worker->pool;
  21480. - /* this can only happen on the local cpu */
  21481. - if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id()))
  21482. - return NULL;
  21483. + if (WARN_ON_ONCE(worker->sleeping))
  21484. + return;
  21485. +
  21486. + worker->sleeping = 1;
  21487. /*
  21488. * The counterpart of the following dec_and_test, implied mb,
  21489. * worklist not empty test sequence is in insert_work().
  21490. * Please read comment there.
  21491. - *
  21492. - * NOT_RUNNING is clear. This means that we're bound to and
  21493. - * running on the local cpu w/ rq lock held and preemption
  21494. - * disabled, which in turn means that none else could be
  21495. - * manipulating idle_list, so dereferencing idle_list without pool
  21496. - * lock is safe.
  21497. */
  21498. if (atomic_dec_and_test(&pool->nr_running) &&
  21499. - !list_empty(&pool->worklist))
  21500. - to_wakeup = first_idle_worker(pool);
  21501. - return to_wakeup ? to_wakeup->task : NULL;
  21502. + !list_empty(&pool->worklist)) {
  21503. + sched_lock_idle_list(pool);
  21504. + wake_up_worker(pool);
  21505. + sched_unlock_idle_list(pool);
  21506. + }
  21507. }
  21508. /**
  21509. @@ -1124,12 +1150,12 @@
  21510. {
  21511. if (pwq) {
  21512. /*
  21513. - * As both pwqs and pools are sched-RCU protected, the
  21514. + * As both pwqs and pools are RCU protected, the
  21515. * following lock operations are safe.
  21516. */
  21517. - spin_lock_irq(&pwq->pool->lock);
  21518. + local_spin_lock_irq(pendingb_lock, &pwq->pool->lock);
  21519. put_pwq(pwq);
  21520. - spin_unlock_irq(&pwq->pool->lock);
  21521. + local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock);
  21522. }
  21523. }
  21524. @@ -1233,7 +1259,7 @@
  21525. struct worker_pool *pool;
  21526. struct pool_workqueue *pwq;
  21527. - local_irq_save(*flags);
  21528. + local_lock_irqsave(pendingb_lock, *flags);
  21529. /* try to steal the timer if it exists */
  21530. if (is_dwork) {
  21531. @@ -1252,6 +1278,7 @@
  21532. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
  21533. return 0;
  21534. + rcu_read_lock();
  21535. /*
  21536. * The queueing is in progress, or it is already queued. Try to
  21537. * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
  21538. @@ -1290,14 +1317,16 @@
  21539. set_work_pool_and_keep_pending(work, pool->id);
  21540. spin_unlock(&pool->lock);
  21541. + rcu_read_unlock();
  21542. return 1;
  21543. }
  21544. spin_unlock(&pool->lock);
  21545. fail:
  21546. - local_irq_restore(*flags);
  21547. + rcu_read_unlock();
  21548. + local_unlock_irqrestore(pendingb_lock, *flags);
  21549. if (work_is_canceling(work))
  21550. return -ENOENT;
  21551. - cpu_relax();
  21552. + cpu_chill();
  21553. return -EAGAIN;
  21554. }
  21555. @@ -1399,7 +1428,7 @@
  21556. * queued or lose PENDING. Grabbing PENDING and queueing should
  21557. * happen with IRQ disabled.
  21558. */
  21559. - WARN_ON_ONCE(!irqs_disabled());
  21560. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  21561. debug_work_activate(work);
  21562. @@ -1407,6 +1436,7 @@
  21563. if (unlikely(wq->flags & __WQ_DRAINING) &&
  21564. WARN_ON_ONCE(!is_chained_work(wq)))
  21565. return;
  21566. + rcu_read_lock();
  21567. retry:
  21568. if (req_cpu == WORK_CPU_UNBOUND)
  21569. cpu = wq_select_unbound_cpu(raw_smp_processor_id());
  21570. @@ -1463,10 +1493,8 @@
  21571. /* pwq determined, queue */
  21572. trace_workqueue_queue_work(req_cpu, pwq, work);
  21573. - if (WARN_ON(!list_empty(&work->entry))) {
  21574. - spin_unlock(&pwq->pool->lock);
  21575. - return;
  21576. - }
  21577. + if (WARN_ON(!list_empty(&work->entry)))
  21578. + goto out;
  21579. pwq->nr_in_flight[pwq->work_color]++;
  21580. work_flags = work_color_to_flags(pwq->work_color);
  21581. @@ -1484,7 +1512,9 @@
  21582. insert_work(pwq, work, worklist, work_flags);
  21583. +out:
  21584. spin_unlock(&pwq->pool->lock);
  21585. + rcu_read_unlock();
  21586. }
  21587. /**
  21588. @@ -1504,14 +1534,14 @@
  21589. bool ret = false;
  21590. unsigned long flags;
  21591. - local_irq_save(flags);
  21592. + local_lock_irqsave(pendingb_lock,flags);
  21593. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  21594. __queue_work(cpu, wq, work);
  21595. ret = true;
  21596. }
  21597. - local_irq_restore(flags);
  21598. + local_unlock_irqrestore(pendingb_lock, flags);
  21599. return ret;
  21600. }
  21601. EXPORT_SYMBOL(queue_work_on);
  21602. @@ -1578,14 +1608,14 @@
  21603. unsigned long flags;
  21604. /* read the comment in __queue_work() */
  21605. - local_irq_save(flags);
  21606. + local_lock_irqsave(pendingb_lock, flags);
  21607. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  21608. __queue_delayed_work(cpu, wq, dwork, delay);
  21609. ret = true;
  21610. }
  21611. - local_irq_restore(flags);
  21612. + local_unlock_irqrestore(pendingb_lock, flags);
  21613. return ret;
  21614. }
  21615. EXPORT_SYMBOL(queue_delayed_work_on);
  21616. @@ -1620,7 +1650,7 @@
  21617. if (likely(ret >= 0)) {
  21618. __queue_delayed_work(cpu, wq, dwork, delay);
  21619. - local_irq_restore(flags);
  21620. + local_unlock_irqrestore(pendingb_lock, flags);
  21621. }
  21622. /* -ENOENT from try_to_grab_pending() becomes %true */
  21623. @@ -1653,7 +1683,9 @@
  21624. worker->last_active = jiffies;
  21625. /* idle_list is LIFO */
  21626. + rt_lock_idle_list(pool);
  21627. list_add(&worker->entry, &pool->idle_list);
  21628. + rt_unlock_idle_list(pool);
  21629. if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
  21630. mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
  21631. @@ -1686,7 +1718,9 @@
  21632. return;
  21633. worker_clr_flags(worker, WORKER_IDLE);
  21634. pool->nr_idle--;
  21635. + rt_lock_idle_list(pool);
  21636. list_del_init(&worker->entry);
  21637. + rt_unlock_idle_list(pool);
  21638. }
  21639. static struct worker *alloc_worker(int node)
  21640. @@ -1852,7 +1886,9 @@
  21641. pool->nr_workers--;
  21642. pool->nr_idle--;
  21643. + rt_lock_idle_list(pool);
  21644. list_del_init(&worker->entry);
  21645. + rt_unlock_idle_list(pool);
  21646. worker->flags |= WORKER_DIE;
  21647. wake_up_process(worker->task);
  21648. }
  21649. @@ -2811,14 +2847,14 @@
  21650. might_sleep();
  21651. - local_irq_disable();
  21652. + rcu_read_lock();
  21653. pool = get_work_pool(work);
  21654. if (!pool) {
  21655. - local_irq_enable();
  21656. + rcu_read_unlock();
  21657. return false;
  21658. }
  21659. - spin_lock(&pool->lock);
  21660. + spin_lock_irq(&pool->lock);
  21661. /* see the comment in try_to_grab_pending() with the same code */
  21662. pwq = get_work_pwq(work);
  21663. if (pwq) {
  21664. @@ -2847,10 +2883,11 @@
  21665. else
  21666. lock_map_acquire_read(&pwq->wq->lockdep_map);
  21667. lock_map_release(&pwq->wq->lockdep_map);
  21668. -
  21669. + rcu_read_unlock();
  21670. return true;
  21671. already_gone:
  21672. spin_unlock_irq(&pool->lock);
  21673. + rcu_read_unlock();
  21674. return false;
  21675. }
  21676. @@ -2937,7 +2974,7 @@
  21677. /* tell other tasks trying to grab @work to back off */
  21678. mark_work_canceling(work);
  21679. - local_irq_restore(flags);
  21680. + local_unlock_irqrestore(pendingb_lock, flags);
  21681. flush_work(work);
  21682. clear_work_data(work);
  21683. @@ -2992,10 +3029,10 @@
  21684. */
  21685. bool flush_delayed_work(struct delayed_work *dwork)
  21686. {
  21687. - local_irq_disable();
  21688. + local_lock_irq(pendingb_lock);
  21689. if (del_timer_sync(&dwork->timer))
  21690. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  21691. - local_irq_enable();
  21692. + local_unlock_irq(pendingb_lock);
  21693. return flush_work(&dwork->work);
  21694. }
  21695. EXPORT_SYMBOL(flush_delayed_work);
  21696. @@ -3030,7 +3067,7 @@
  21697. set_work_pool_and_clear_pending(&dwork->work,
  21698. get_work_pool_id(&dwork->work));
  21699. - local_irq_restore(flags);
  21700. + local_unlock_irqrestore(pendingb_lock, flags);
  21701. return ret;
  21702. }
  21703. EXPORT_SYMBOL(cancel_delayed_work);
  21704. @@ -3259,7 +3296,7 @@
  21705. * put_unbound_pool - put a worker_pool
  21706. * @pool: worker_pool to put
  21707. *
  21708. - * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
  21709. + * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
  21710. * safe manner. get_unbound_pool() calls this function on its failure path
  21711. * and this function should be able to release pools which went through,
  21712. * successfully or not, init_worker_pool().
  21713. @@ -3313,8 +3350,8 @@
  21714. del_timer_sync(&pool->idle_timer);
  21715. del_timer_sync(&pool->mayday_timer);
  21716. - /* sched-RCU protected to allow dereferences from get_work_pool() */
  21717. - call_rcu_sched(&pool->rcu, rcu_free_pool);
  21718. + /* RCU protected to allow dereferences from get_work_pool() */
  21719. + call_rcu(&pool->rcu, rcu_free_pool);
  21720. }
  21721. /**
  21722. @@ -3421,14 +3458,14 @@
  21723. put_unbound_pool(pool);
  21724. mutex_unlock(&wq_pool_mutex);
  21725. - call_rcu_sched(&pwq->rcu, rcu_free_pwq);
  21726. + call_rcu(&pwq->rcu, rcu_free_pwq);
  21727. /*
  21728. * If we're the last pwq going away, @wq is already dead and no one
  21729. * is gonna access it anymore. Schedule RCU free.
  21730. */
  21731. if (is_last)
  21732. - call_rcu_sched(&wq->rcu, rcu_free_wq);
  21733. + call_rcu(&wq->rcu, rcu_free_wq);
  21734. }
  21735. /**
  21736. @@ -4078,7 +4115,7 @@
  21737. * The base ref is never dropped on per-cpu pwqs. Directly
  21738. * schedule RCU free.
  21739. */
  21740. - call_rcu_sched(&wq->rcu, rcu_free_wq);
  21741. + call_rcu(&wq->rcu, rcu_free_wq);
  21742. } else {
  21743. /*
  21744. * We're the sole accessor of @wq at this point. Directly
  21745. @@ -4171,7 +4208,8 @@
  21746. struct pool_workqueue *pwq;
  21747. bool ret;
  21748. - rcu_read_lock_sched();
  21749. + rcu_read_lock();
  21750. + preempt_disable();
  21751. if (cpu == WORK_CPU_UNBOUND)
  21752. cpu = smp_processor_id();
  21753. @@ -4182,7 +4220,8 @@
  21754. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  21755. ret = !list_empty(&pwq->delayed_works);
  21756. - rcu_read_unlock_sched();
  21757. + preempt_enable();
  21758. + rcu_read_unlock();
  21759. return ret;
  21760. }
  21761. @@ -4208,15 +4247,15 @@
  21762. if (work_pending(work))
  21763. ret |= WORK_BUSY_PENDING;
  21764. - local_irq_save(flags);
  21765. + rcu_read_lock();
  21766. pool = get_work_pool(work);
  21767. if (pool) {
  21768. - spin_lock(&pool->lock);
  21769. + spin_lock_irqsave(&pool->lock, flags);
  21770. if (find_worker_executing_work(pool, work))
  21771. ret |= WORK_BUSY_RUNNING;
  21772. - spin_unlock(&pool->lock);
  21773. + spin_unlock_irqrestore(&pool->lock, flags);
  21774. }
  21775. - local_irq_restore(flags);
  21776. + rcu_read_unlock();
  21777. return ret;
  21778. }
  21779. @@ -4405,7 +4444,7 @@
  21780. unsigned long flags;
  21781. int pi;
  21782. - rcu_read_lock_sched();
  21783. + rcu_read_lock();
  21784. pr_info("Showing busy workqueues and worker pools:\n");
  21785. @@ -4458,7 +4497,7 @@
  21786. spin_unlock_irqrestore(&pool->lock, flags);
  21787. }
  21788. - rcu_read_unlock_sched();
  21789. + rcu_read_unlock();
  21790. }
  21791. /*
  21792. @@ -4819,16 +4858,16 @@
  21793. * nr_active is monotonically decreasing. It's safe
  21794. * to peek without lock.
  21795. */
  21796. - rcu_read_lock_sched();
  21797. + rcu_read_lock();
  21798. for_each_pwq(pwq, wq) {
  21799. WARN_ON_ONCE(pwq->nr_active < 0);
  21800. if (pwq->nr_active) {
  21801. busy = true;
  21802. - rcu_read_unlock_sched();
  21803. + rcu_read_unlock();
  21804. goto out_unlock;
  21805. }
  21806. }
  21807. - rcu_read_unlock_sched();
  21808. + rcu_read_unlock();
  21809. }
  21810. out_unlock:
  21811. mutex_unlock(&wq_pool_mutex);
  21812. @@ -5018,7 +5057,8 @@
  21813. const char *delim = "";
  21814. int node, written = 0;
  21815. - rcu_read_lock_sched();
  21816. + get_online_cpus();
  21817. + rcu_read_lock();
  21818. for_each_node(node) {
  21819. written += scnprintf(buf + written, PAGE_SIZE - written,
  21820. "%s%d:%d", delim, node,
  21821. @@ -5026,7 +5066,8 @@
  21822. delim = " ";
  21823. }
  21824. written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
  21825. - rcu_read_unlock_sched();
  21826. + rcu_read_unlock();
  21827. + put_online_cpus();
  21828. return written;
  21829. }
  21830. diff -Nur linux-4.6.6.orig/kernel/workqueue_internal.h linux-4.6.6/kernel/workqueue_internal.h
  21831. --- linux-4.6.6.orig/kernel/workqueue_internal.h 2016-08-10 12:54:57.000000000 +0200
  21832. +++ linux-4.6.6/kernel/workqueue_internal.h 2016-08-22 21:27:47.433894394 +0200
  21833. @@ -43,6 +43,7 @@
  21834. unsigned long last_active; /* L: last active timestamp */
  21835. unsigned int flags; /* X: flags */
  21836. int id; /* I: worker id */
  21837. + int sleeping; /* None */
  21838. /*
  21839. * Opaque string set with work_set_desc(). Printed out with task
  21840. @@ -68,7 +69,7 @@
  21841. * Scheduler hooks for concurrency managed workqueue. Only to be used from
  21842. * sched/core.c and workqueue.c.
  21843. */
  21844. -void wq_worker_waking_up(struct task_struct *task, int cpu);
  21845. -struct task_struct *wq_worker_sleeping(struct task_struct *task);
  21846. +void wq_worker_running(struct task_struct *task);
  21847. +void wq_worker_sleeping(struct task_struct *task);
  21848. #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */
  21849. diff -Nur linux-4.6.6.orig/lib/debugobjects.c linux-4.6.6/lib/debugobjects.c
  21850. --- linux-4.6.6.orig/lib/debugobjects.c 2016-08-10 12:54:57.000000000 +0200
  21851. +++ linux-4.6.6/lib/debugobjects.c 2016-08-22 21:27:47.433894394 +0200
  21852. @@ -309,7 +309,10 @@
  21853. struct debug_obj *obj;
  21854. unsigned long flags;
  21855. - fill_pool();
  21856. +#ifdef CONFIG_PREEMPT_RT_FULL
  21857. + if (preempt_count() == 0 && !irqs_disabled())
  21858. +#endif
  21859. + fill_pool();
  21860. db = get_bucket((unsigned long) addr);
  21861. diff -Nur linux-4.6.6.orig/lib/idr.c linux-4.6.6/lib/idr.c
  21862. --- linux-4.6.6.orig/lib/idr.c 2016-08-10 12:54:57.000000000 +0200
  21863. +++ linux-4.6.6/lib/idr.c 2016-08-22 21:27:47.433894394 +0200
  21864. @@ -30,6 +30,7 @@
  21865. #include <linux/idr.h>
  21866. #include <linux/spinlock.h>
  21867. #include <linux/percpu.h>
  21868. +#include <linux/locallock.h>
  21869. #define MAX_IDR_SHIFT (sizeof(int) * 8 - 1)
  21870. #define MAX_IDR_BIT (1U << MAX_IDR_SHIFT)
  21871. @@ -45,6 +46,37 @@
  21872. static DEFINE_PER_CPU(int, idr_preload_cnt);
  21873. static DEFINE_SPINLOCK(simple_ida_lock);
  21874. +#ifdef CONFIG_PREEMPT_RT_FULL
  21875. +static DEFINE_LOCAL_IRQ_LOCK(idr_lock);
  21876. +
  21877. +static inline void idr_preload_lock(void)
  21878. +{
  21879. + local_lock(idr_lock);
  21880. +}
  21881. +
  21882. +static inline void idr_preload_unlock(void)
  21883. +{
  21884. + local_unlock(idr_lock);
  21885. +}
  21886. +
  21887. +void idr_preload_end(void)
  21888. +{
  21889. + idr_preload_unlock();
  21890. +}
  21891. +EXPORT_SYMBOL(idr_preload_end);
  21892. +#else
  21893. +static inline void idr_preload_lock(void)
  21894. +{
  21895. + preempt_disable();
  21896. +}
  21897. +
  21898. +static inline void idr_preload_unlock(void)
  21899. +{
  21900. + preempt_enable();
  21901. +}
  21902. +#endif
  21903. +
  21904. +
  21905. /* the maximum ID which can be allocated given idr->layers */
  21906. static int idr_max(int layers)
  21907. {
  21908. @@ -115,14 +147,14 @@
  21909. * context. See idr_preload() for details.
  21910. */
  21911. if (!in_interrupt()) {
  21912. - preempt_disable();
  21913. + idr_preload_lock();
  21914. new = __this_cpu_read(idr_preload_head);
  21915. if (new) {
  21916. __this_cpu_write(idr_preload_head, new->ary[0]);
  21917. __this_cpu_dec(idr_preload_cnt);
  21918. new->ary[0] = NULL;
  21919. }
  21920. - preempt_enable();
  21921. + idr_preload_unlock();
  21922. if (new)
  21923. return new;
  21924. }
  21925. @@ -366,7 +398,6 @@
  21926. idr_mark_full(pa, id);
  21927. }
  21928. -
  21929. /**
  21930. * idr_preload - preload for idr_alloc()
  21931. * @gfp_mask: allocation mask to use for preloading
  21932. @@ -401,7 +432,7 @@
  21933. WARN_ON_ONCE(in_interrupt());
  21934. might_sleep_if(gfpflags_allow_blocking(gfp_mask));
  21935. - preempt_disable();
  21936. + idr_preload_lock();
  21937. /*
  21938. * idr_alloc() is likely to succeed w/o full idr_layer buffer and
  21939. @@ -413,9 +444,9 @@
  21940. while (__this_cpu_read(idr_preload_cnt) < MAX_IDR_FREE) {
  21941. struct idr_layer *new;
  21942. - preempt_enable();
  21943. + idr_preload_unlock();
  21944. new = kmem_cache_zalloc(idr_layer_cache, gfp_mask);
  21945. - preempt_disable();
  21946. + idr_preload_lock();
  21947. if (!new)
  21948. break;
  21949. diff -Nur linux-4.6.6.orig/lib/irq_poll.c linux-4.6.6/lib/irq_poll.c
  21950. --- linux-4.6.6.orig/lib/irq_poll.c 2016-08-10 12:54:57.000000000 +0200
  21951. +++ linux-4.6.6/lib/irq_poll.c 2016-08-22 21:27:47.433894394 +0200
  21952. @@ -36,6 +36,7 @@
  21953. list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
  21954. __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
  21955. local_irq_restore(flags);
  21956. + preempt_check_resched_rt();
  21957. }
  21958. EXPORT_SYMBOL(irq_poll_sched);
  21959. @@ -71,6 +72,7 @@
  21960. local_irq_save(flags);
  21961. __irq_poll_complete(iop);
  21962. local_irq_restore(flags);
  21963. + preempt_check_resched_rt();
  21964. }
  21965. EXPORT_SYMBOL(irq_poll_complete);
  21966. @@ -95,6 +97,7 @@
  21967. }
  21968. local_irq_enable();
  21969. + preempt_check_resched_rt();
  21970. /* Even though interrupts have been re-enabled, this
  21971. * access is safe because interrupts can only add new
  21972. @@ -132,6 +135,7 @@
  21973. __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
  21974. local_irq_enable();
  21975. + preempt_check_resched_rt();
  21976. }
  21977. /**
  21978. @@ -199,6 +203,7 @@
  21979. this_cpu_ptr(&blk_cpu_iopoll));
  21980. __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
  21981. local_irq_enable();
  21982. + preempt_check_resched_rt();
  21983. }
  21984. return NOTIFY_OK;
  21985. diff -Nur linux-4.6.6.orig/lib/Kconfig linux-4.6.6/lib/Kconfig
  21986. --- linux-4.6.6.orig/lib/Kconfig 2016-08-10 12:54:57.000000000 +0200
  21987. +++ linux-4.6.6/lib/Kconfig 2016-08-22 21:27:47.433894394 +0200
  21988. @@ -397,6 +397,7 @@
  21989. config CPUMASK_OFFSTACK
  21990. bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS
  21991. + depends on !PREEMPT_RT_FULL
  21992. help
  21993. Use dynamic allocation for cpumask_var_t, instead of putting
  21994. them on the stack. This is a bit more expensive, but avoids
  21995. diff -Nur linux-4.6.6.orig/lib/locking-selftest.c linux-4.6.6/lib/locking-selftest.c
  21996. --- linux-4.6.6.orig/lib/locking-selftest.c 2016-08-10 12:54:57.000000000 +0200
  21997. +++ linux-4.6.6/lib/locking-selftest.c 2016-08-22 21:27:47.433894394 +0200
  21998. @@ -590,6 +590,8 @@
  21999. #include "locking-selftest-spin-hardirq.h"
  22000. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
  22001. +#ifndef CONFIG_PREEMPT_RT_FULL
  22002. +
  22003. #include "locking-selftest-rlock-hardirq.h"
  22004. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
  22005. @@ -605,9 +607,12 @@
  22006. #include "locking-selftest-wlock-softirq.h"
  22007. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
  22008. +#endif
  22009. +
  22010. #undef E1
  22011. #undef E2
  22012. +#ifndef CONFIG_PREEMPT_RT_FULL
  22013. /*
  22014. * Enabling hardirqs with a softirq-safe lock held:
  22015. */
  22016. @@ -640,6 +645,8 @@
  22017. #undef E1
  22018. #undef E2
  22019. +#endif
  22020. +
  22021. /*
  22022. * Enabling irqs with an irq-safe lock held:
  22023. */
  22024. @@ -663,6 +670,8 @@
  22025. #include "locking-selftest-spin-hardirq.h"
  22026. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
  22027. +#ifndef CONFIG_PREEMPT_RT_FULL
  22028. +
  22029. #include "locking-selftest-rlock-hardirq.h"
  22030. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
  22031. @@ -678,6 +687,8 @@
  22032. #include "locking-selftest-wlock-softirq.h"
  22033. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
  22034. +#endif
  22035. +
  22036. #undef E1
  22037. #undef E2
  22038. @@ -709,6 +720,8 @@
  22039. #include "locking-selftest-spin-hardirq.h"
  22040. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
  22041. +#ifndef CONFIG_PREEMPT_RT_FULL
  22042. +
  22043. #include "locking-selftest-rlock-hardirq.h"
  22044. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
  22045. @@ -724,6 +737,8 @@
  22046. #include "locking-selftest-wlock-softirq.h"
  22047. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
  22048. +#endif
  22049. +
  22050. #undef E1
  22051. #undef E2
  22052. #undef E3
  22053. @@ -757,6 +772,8 @@
  22054. #include "locking-selftest-spin-hardirq.h"
  22055. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
  22056. +#ifndef CONFIG_PREEMPT_RT_FULL
  22057. +
  22058. #include "locking-selftest-rlock-hardirq.h"
  22059. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
  22060. @@ -772,10 +789,14 @@
  22061. #include "locking-selftest-wlock-softirq.h"
  22062. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
  22063. +#endif
  22064. +
  22065. #undef E1
  22066. #undef E2
  22067. #undef E3
  22068. +#ifndef CONFIG_PREEMPT_RT_FULL
  22069. +
  22070. /*
  22071. * read-lock / write-lock irq inversion.
  22072. *
  22073. @@ -838,6 +859,10 @@
  22074. #undef E2
  22075. #undef E3
  22076. +#endif
  22077. +
  22078. +#ifndef CONFIG_PREEMPT_RT_FULL
  22079. +
  22080. /*
  22081. * read-lock / write-lock recursion that is actually safe.
  22082. */
  22083. @@ -876,6 +901,8 @@
  22084. #undef E2
  22085. #undef E3
  22086. +#endif
  22087. +
  22088. /*
  22089. * read-lock / write-lock recursion that is unsafe.
  22090. */
  22091. @@ -1858,6 +1885,7 @@
  22092. printk(" --------------------------------------------------------------------------\n");
  22093. +#ifndef CONFIG_PREEMPT_RT_FULL
  22094. /*
  22095. * irq-context testcases:
  22096. */
  22097. @@ -1870,6 +1898,28 @@
  22098. DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion);
  22099. // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2);
  22100. +#else
  22101. + /* On -rt, we only do hardirq context test for raw spinlock */
  22102. + DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 12);
  22103. + DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 21);
  22104. +
  22105. + DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 12);
  22106. + DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 21);
  22107. +
  22108. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 123);
  22109. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 132);
  22110. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 213);
  22111. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 231);
  22112. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 312);
  22113. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 321);
  22114. +
  22115. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 123);
  22116. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 132);
  22117. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 213);
  22118. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 231);
  22119. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 312);
  22120. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 321);
  22121. +#endif
  22122. ww_tests();
  22123. diff -Nur linux-4.6.6.orig/lib/percpu_ida.c linux-4.6.6/lib/percpu_ida.c
  22124. --- linux-4.6.6.orig/lib/percpu_ida.c 2016-08-10 12:54:57.000000000 +0200
  22125. +++ linux-4.6.6/lib/percpu_ida.c 2016-08-22 21:27:47.433894394 +0200
  22126. @@ -26,6 +26,9 @@
  22127. #include <linux/string.h>
  22128. #include <linux/spinlock.h>
  22129. #include <linux/percpu_ida.h>
  22130. +#include <linux/locallock.h>
  22131. +
  22132. +static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock);
  22133. struct percpu_ida_cpu {
  22134. /*
  22135. @@ -148,13 +151,13 @@
  22136. unsigned long flags;
  22137. int tag;
  22138. - local_irq_save(flags);
  22139. + local_lock_irqsave(irq_off_lock, flags);
  22140. tags = this_cpu_ptr(pool->tag_cpu);
  22141. /* Fastpath */
  22142. tag = alloc_local_tag(tags);
  22143. if (likely(tag >= 0)) {
  22144. - local_irq_restore(flags);
  22145. + local_unlock_irqrestore(irq_off_lock, flags);
  22146. return tag;
  22147. }
  22148. @@ -173,6 +176,7 @@
  22149. if (!tags->nr_free)
  22150. alloc_global_tags(pool, tags);
  22151. +
  22152. if (!tags->nr_free)
  22153. steal_tags(pool, tags);
  22154. @@ -184,7 +188,7 @@
  22155. }
  22156. spin_unlock(&pool->lock);
  22157. - local_irq_restore(flags);
  22158. + local_unlock_irqrestore(irq_off_lock, flags);
  22159. if (tag >= 0 || state == TASK_RUNNING)
  22160. break;
  22161. @@ -196,7 +200,7 @@
  22162. schedule();
  22163. - local_irq_save(flags);
  22164. + local_lock_irqsave(irq_off_lock, flags);
  22165. tags = this_cpu_ptr(pool->tag_cpu);
  22166. }
  22167. if (state != TASK_RUNNING)
  22168. @@ -221,7 +225,7 @@
  22169. BUG_ON(tag >= pool->nr_tags);
  22170. - local_irq_save(flags);
  22171. + local_lock_irqsave(irq_off_lock, flags);
  22172. tags = this_cpu_ptr(pool->tag_cpu);
  22173. spin_lock(&tags->lock);
  22174. @@ -253,7 +257,7 @@
  22175. spin_unlock(&pool->lock);
  22176. }
  22177. - local_irq_restore(flags);
  22178. + local_unlock_irqrestore(irq_off_lock, flags);
  22179. }
  22180. EXPORT_SYMBOL_GPL(percpu_ida_free);
  22181. @@ -345,7 +349,7 @@
  22182. struct percpu_ida_cpu *remote;
  22183. unsigned cpu, i, err = 0;
  22184. - local_irq_save(flags);
  22185. + local_lock_irqsave(irq_off_lock, flags);
  22186. for_each_possible_cpu(cpu) {
  22187. remote = per_cpu_ptr(pool->tag_cpu, cpu);
  22188. spin_lock(&remote->lock);
  22189. @@ -367,7 +371,7 @@
  22190. }
  22191. spin_unlock(&pool->lock);
  22192. out:
  22193. - local_irq_restore(flags);
  22194. + local_unlock_irqrestore(irq_off_lock, flags);
  22195. return err;
  22196. }
  22197. EXPORT_SYMBOL_GPL(percpu_ida_for_each_free);
  22198. diff -Nur linux-4.6.6.orig/lib/radix-tree.c linux-4.6.6/lib/radix-tree.c
  22199. --- linux-4.6.6.orig/lib/radix-tree.c 2016-08-10 12:54:57.000000000 +0200
  22200. +++ linux-4.6.6/lib/radix-tree.c 2016-08-22 21:27:47.433894394 +0200
  22201. @@ -240,13 +240,14 @@
  22202. * succeed in getting a node here (and never reach
  22203. * kmem_cache_alloc)
  22204. */
  22205. - rtp = this_cpu_ptr(&radix_tree_preloads);
  22206. + rtp = &get_cpu_var(radix_tree_preloads);
  22207. if (rtp->nr) {
  22208. ret = rtp->nodes;
  22209. rtp->nodes = ret->private_data;
  22210. ret->private_data = NULL;
  22211. rtp->nr--;
  22212. }
  22213. + put_cpu_var(radix_tree_preloads);
  22214. /*
  22215. * Update the allocation stack trace as this is more useful
  22216. * for debugging.
  22217. @@ -287,6 +288,7 @@
  22218. call_rcu(&node->rcu_head, radix_tree_node_rcu_free);
  22219. }
  22220. +#ifndef CONFIG_PREEMPT_RT_FULL
  22221. /*
  22222. * Load up this CPU's radix_tree_node buffer with sufficient objects to
  22223. * ensure that the addition of a single element in the tree cannot fail. On
  22224. @@ -355,6 +357,7 @@
  22225. return 0;
  22226. }
  22227. EXPORT_SYMBOL(radix_tree_maybe_preload);
  22228. +#endif
  22229. /*
  22230. * Return the maximum key which can be store into a
  22231. diff -Nur linux-4.6.6.orig/lib/random32.c linux-4.6.6/lib/random32.c
  22232. --- linux-4.6.6.orig/lib/random32.c 2016-08-10 12:54:57.000000000 +0200
  22233. +++ linux-4.6.6/lib/random32.c 2016-08-22 21:27:47.433894394 +0200
  22234. @@ -233,7 +233,6 @@
  22235. static void __init __prandom_start_seed_timer(void)
  22236. {
  22237. - set_timer_slack(&seed_timer, HZ);
  22238. seed_timer.expires = jiffies + msecs_to_jiffies(40 * MSEC_PER_SEC);
  22239. add_timer(&seed_timer);
  22240. }
  22241. diff -Nur linux-4.6.6.orig/lib/rbtree.c linux-4.6.6/lib/rbtree.c
  22242. --- linux-4.6.6.orig/lib/rbtree.c 2016-08-10 12:54:57.000000000 +0200
  22243. +++ linux-4.6.6/lib/rbtree.c 2016-08-22 21:27:47.437894654 +0200
  22244. @@ -23,6 +23,7 @@
  22245. #include <linux/rbtree_augmented.h>
  22246. #include <linux/export.h>
  22247. +#include <linux/rcupdate.h>
  22248. /*
  22249. * red-black trees properties: http://en.wikipedia.org/wiki/Rbtree
  22250. @@ -590,3 +591,13 @@
  22251. return rb_left_deepest_node(root->rb_node);
  22252. }
  22253. EXPORT_SYMBOL(rb_first_postorder);
  22254. +
  22255. +void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,
  22256. + struct rb_node **rb_link)
  22257. +{
  22258. + node->__rb_parent_color = (unsigned long)parent;
  22259. + node->rb_left = node->rb_right = NULL;
  22260. +
  22261. + rcu_assign_pointer(*rb_link, node);
  22262. +}
  22263. +EXPORT_SYMBOL(rb_link_node_rcu);
  22264. diff -Nur linux-4.6.6.orig/lib/scatterlist.c linux-4.6.6/lib/scatterlist.c
  22265. --- linux-4.6.6.orig/lib/scatterlist.c 2016-08-10 12:54:57.000000000 +0200
  22266. +++ linux-4.6.6/lib/scatterlist.c 2016-08-22 21:27:47.437894654 +0200
  22267. @@ -620,7 +620,7 @@
  22268. flush_kernel_dcache_page(miter->page);
  22269. if (miter->__flags & SG_MITER_ATOMIC) {
  22270. - WARN_ON_ONCE(preemptible());
  22271. + WARN_ON_ONCE(!pagefault_disabled());
  22272. kunmap_atomic(miter->addr);
  22273. } else
  22274. kunmap(miter->page);
  22275. @@ -664,7 +664,7 @@
  22276. if (!sg_miter_skip(&miter, skip))
  22277. return false;
  22278. - local_irq_save(flags);
  22279. + local_irq_save_nort(flags);
  22280. while (sg_miter_next(&miter) && offset < buflen) {
  22281. unsigned int len;
  22282. @@ -681,7 +681,7 @@
  22283. sg_miter_stop(&miter);
  22284. - local_irq_restore(flags);
  22285. + local_irq_restore_nort(flags);
  22286. return offset;
  22287. }
  22288. EXPORT_SYMBOL(sg_copy_buffer);
  22289. diff -Nur linux-4.6.6.orig/lib/smp_processor_id.c linux-4.6.6/lib/smp_processor_id.c
  22290. --- linux-4.6.6.orig/lib/smp_processor_id.c 2016-08-10 12:54:57.000000000 +0200
  22291. +++ linux-4.6.6/lib/smp_processor_id.c 2016-08-22 21:27:47.437894654 +0200
  22292. @@ -39,8 +39,9 @@
  22293. if (!printk_ratelimit())
  22294. goto out_enable;
  22295. - printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n",
  22296. - what1, what2, preempt_count() - 1, current->comm, current->pid);
  22297. + printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n",
  22298. + what1, what2, preempt_count() - 1, __migrate_disabled(current),
  22299. + current->comm, current->pid);
  22300. print_symbol("caller is %s\n", (long)__builtin_return_address(0));
  22301. dump_stack();
  22302. diff -Nur linux-4.6.6.orig/localversion-rt linux-4.6.6/localversion-rt
  22303. --- linux-4.6.6.orig/localversion-rt 1970-01-01 01:00:00.000000000 +0100
  22304. +++ linux-4.6.6/localversion-rt 2016-08-22 21:27:47.437894654 +0200
  22305. @@ -0,0 +1 @@
  22306. +-rt10
  22307. diff -Nur linux-4.6.6.orig/mm/backing-dev.c linux-4.6.6/mm/backing-dev.c
  22308. --- linux-4.6.6.orig/mm/backing-dev.c 2016-08-10 12:54:57.000000000 +0200
  22309. +++ linux-4.6.6/mm/backing-dev.c 2016-08-22 21:27:47.437894654 +0200
  22310. @@ -457,9 +457,9 @@
  22311. {
  22312. unsigned long flags;
  22313. - local_irq_save(flags);
  22314. + local_irq_save_nort(flags);
  22315. if (!atomic_dec_and_lock(&congested->refcnt, &cgwb_lock)) {
  22316. - local_irq_restore(flags);
  22317. + local_irq_restore_nort(flags);
  22318. return;
  22319. }
  22320. diff -Nur linux-4.6.6.orig/mm/compaction.c linux-4.6.6/mm/compaction.c
  22321. --- linux-4.6.6.orig/mm/compaction.c 2016-08-10 12:54:57.000000000 +0200
  22322. +++ linux-4.6.6/mm/compaction.c 2016-08-22 21:27:47.437894654 +0200
  22323. @@ -1409,10 +1409,12 @@
  22324. cc->migrate_pfn & ~((1UL << cc->order) - 1);
  22325. if (cc->last_migrated_pfn < current_block_start) {
  22326. - cpu = get_cpu();
  22327. + cpu = get_cpu_light();
  22328. + local_lock_irq(swapvec_lock);
  22329. lru_add_drain_cpu(cpu);
  22330. + local_unlock_irq(swapvec_lock);
  22331. drain_local_pages(zone);
  22332. - put_cpu();
  22333. + put_cpu_light();
  22334. /* No more flushing until we migrate again */
  22335. cc->last_migrated_pfn = 0;
  22336. }
  22337. diff -Nur linux-4.6.6.orig/mm/filemap.c linux-4.6.6/mm/filemap.c
  22338. --- linux-4.6.6.orig/mm/filemap.c 2016-08-10 12:54:57.000000000 +0200
  22339. +++ linux-4.6.6/mm/filemap.c 2016-08-22 21:27:47.437894654 +0200
  22340. @@ -169,7 +169,9 @@
  22341. if (!workingset_node_pages(node) &&
  22342. list_empty(&node->private_list)) {
  22343. node->private_data = mapping;
  22344. - list_lru_add(&workingset_shadow_nodes, &node->private_list);
  22345. + local_lock(workingset_shadow_lock);
  22346. + list_lru_add(&__workingset_shadow_nodes, &node->private_list);
  22347. + local_unlock(workingset_shadow_lock);
  22348. }
  22349. }
  22350. @@ -618,9 +620,12 @@
  22351. * node->private_list is protected by
  22352. * mapping->tree_lock.
  22353. */
  22354. - if (!list_empty(&node->private_list))
  22355. - list_lru_del(&workingset_shadow_nodes,
  22356. + if (!list_empty(&node->private_list)) {
  22357. + local_lock(workingset_shadow_lock);
  22358. + list_lru_del(&__workingset_shadow_nodes,
  22359. &node->private_list);
  22360. + local_unlock(workingset_shadow_lock);
  22361. + }
  22362. }
  22363. return 0;
  22364. }
  22365. diff -Nur linux-4.6.6.orig/mm/highmem.c linux-4.6.6/mm/highmem.c
  22366. --- linux-4.6.6.orig/mm/highmem.c 2016-08-10 12:54:57.000000000 +0200
  22367. +++ linux-4.6.6/mm/highmem.c 2016-08-22 21:27:47.437894654 +0200
  22368. @@ -29,10 +29,11 @@
  22369. #include <linux/kgdb.h>
  22370. #include <asm/tlbflush.h>
  22371. -
  22372. +#ifndef CONFIG_PREEMPT_RT_FULL
  22373. #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  22374. DEFINE_PER_CPU(int, __kmap_atomic_idx);
  22375. #endif
  22376. +#endif
  22377. /*
  22378. * Virtual_count is not a pure "count".
  22379. @@ -107,8 +108,9 @@
  22380. unsigned long totalhigh_pages __read_mostly;
  22381. EXPORT_SYMBOL(totalhigh_pages);
  22382. -
  22383. +#ifndef CONFIG_PREEMPT_RT_FULL
  22384. EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
  22385. +#endif
  22386. unsigned int nr_free_highpages (void)
  22387. {
  22388. diff -Nur linux-4.6.6.orig/mm/Kconfig linux-4.6.6/mm/Kconfig
  22389. --- linux-4.6.6.orig/mm/Kconfig 2016-08-10 12:54:57.000000000 +0200
  22390. +++ linux-4.6.6/mm/Kconfig 2016-08-22 21:27:47.437894654 +0200
  22391. @@ -391,7 +391,7 @@
  22392. config TRANSPARENT_HUGEPAGE
  22393. bool "Transparent Hugepage Support"
  22394. - depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE
  22395. + depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT_FULL
  22396. select COMPACTION
  22397. help
  22398. Transparent Hugepages allows the kernel to use huge pages and
  22399. diff -Nur linux-4.6.6.orig/mm/memcontrol.c linux-4.6.6/mm/memcontrol.c
  22400. --- linux-4.6.6.orig/mm/memcontrol.c 2016-08-10 12:54:57.000000000 +0200
  22401. +++ linux-4.6.6/mm/memcontrol.c 2016-08-22 21:27:47.441894907 +0200
  22402. @@ -67,6 +67,7 @@
  22403. #include <net/sock.h>
  22404. #include <net/ip.h>
  22405. #include "slab.h"
  22406. +#include <linux/locallock.h>
  22407. #include <asm/uaccess.h>
  22408. @@ -92,6 +93,8 @@
  22409. #define do_swap_account 0
  22410. #endif
  22411. +static DEFINE_LOCAL_IRQ_LOCK(event_lock);
  22412. +
  22413. /* Whether legacy memory+swap accounting is active */
  22414. static bool do_memsw_account(void)
  22415. {
  22416. @@ -1825,14 +1828,17 @@
  22417. */
  22418. static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  22419. {
  22420. - struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
  22421. + struct memcg_stock_pcp *stock;
  22422. + int cpu = get_cpu_light();
  22423. +
  22424. + stock = &per_cpu(memcg_stock, cpu);
  22425. if (stock->cached != memcg) { /* reset if necessary */
  22426. drain_stock(stock);
  22427. stock->cached = memcg;
  22428. }
  22429. stock->nr_pages += nr_pages;
  22430. - put_cpu_var(memcg_stock);
  22431. + put_cpu_light();
  22432. }
  22433. /*
  22434. @@ -1848,7 +1854,7 @@
  22435. return;
  22436. /* Notify other cpus that system-wide "drain" is running */
  22437. get_online_cpus();
  22438. - curcpu = get_cpu();
  22439. + curcpu = get_cpu_light();
  22440. for_each_online_cpu(cpu) {
  22441. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  22442. struct mem_cgroup *memcg;
  22443. @@ -1865,7 +1871,7 @@
  22444. schedule_work_on(cpu, &stock->work);
  22445. }
  22446. }
  22447. - put_cpu();
  22448. + put_cpu_light();
  22449. put_online_cpus();
  22450. mutex_unlock(&percpu_charge_mutex);
  22451. }
  22452. @@ -4551,12 +4557,12 @@
  22453. ret = 0;
  22454. - local_irq_disable();
  22455. + local_lock_irq(event_lock);
  22456. mem_cgroup_charge_statistics(to, page, compound, nr_pages);
  22457. memcg_check_events(to, page);
  22458. mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
  22459. memcg_check_events(from, page);
  22460. - local_irq_enable();
  22461. + local_unlock_irq(event_lock);
  22462. out_unlock:
  22463. unlock_page(page);
  22464. out:
  22465. @@ -5406,10 +5412,10 @@
  22466. commit_charge(page, memcg, lrucare);
  22467. - local_irq_disable();
  22468. + local_lock_irq(event_lock);
  22469. mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
  22470. memcg_check_events(memcg, page);
  22471. - local_irq_enable();
  22472. + local_unlock_irq(event_lock);
  22473. if (do_memsw_account() && PageSwapCache(page)) {
  22474. swp_entry_t entry = { .val = page_private(page) };
  22475. @@ -5461,14 +5467,14 @@
  22476. memcg_oom_recover(memcg);
  22477. }
  22478. - local_irq_save(flags);
  22479. + local_lock_irqsave(event_lock, flags);
  22480. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
  22481. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
  22482. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
  22483. __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
  22484. __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
  22485. memcg_check_events(memcg, dummy_page);
  22486. - local_irq_restore(flags);
  22487. + local_unlock_irqrestore(event_lock, flags);
  22488. if (!mem_cgroup_is_root(memcg))
  22489. css_put_many(&memcg->css, nr_pages);
  22490. @@ -5787,6 +5793,7 @@
  22491. {
  22492. struct mem_cgroup *memcg;
  22493. unsigned short oldid;
  22494. + unsigned long flags;
  22495. VM_BUG_ON_PAGE(PageLRU(page), page);
  22496. VM_BUG_ON_PAGE(page_count(page), page);
  22497. diff -Nur linux-4.6.6.orig/mm/mmu_context.c linux-4.6.6/mm/mmu_context.c
  22498. --- linux-4.6.6.orig/mm/mmu_context.c 2016-08-10 12:54:57.000000000 +0200
  22499. +++ linux-4.6.6/mm/mmu_context.c 2016-08-22 21:27:47.441894907 +0200
  22500. @@ -23,6 +23,7 @@
  22501. struct task_struct *tsk = current;
  22502. task_lock(tsk);
  22503. + preempt_disable_rt();
  22504. active_mm = tsk->active_mm;
  22505. if (active_mm != mm) {
  22506. atomic_inc(&mm->mm_count);
  22507. @@ -30,6 +31,7 @@
  22508. }
  22509. tsk->mm = mm;
  22510. switch_mm(active_mm, mm, tsk);
  22511. + preempt_enable_rt();
  22512. task_unlock(tsk);
  22513. #ifdef finish_arch_post_lock_switch
  22514. finish_arch_post_lock_switch();
  22515. diff -Nur linux-4.6.6.orig/mm/page_alloc.c linux-4.6.6/mm/page_alloc.c
  22516. --- linux-4.6.6.orig/mm/page_alloc.c 2016-08-10 12:54:57.000000000 +0200
  22517. +++ linux-4.6.6/mm/page_alloc.c 2016-08-22 21:27:47.441894907 +0200
  22518. @@ -61,6 +61,7 @@
  22519. #include <linux/page_ext.h>
  22520. #include <linux/hugetlb.h>
  22521. #include <linux/sched/rt.h>
  22522. +#include <linux/locallock.h>
  22523. #include <linux/page_owner.h>
  22524. #include <linux/kthread.h>
  22525. @@ -275,6 +276,18 @@
  22526. EXPORT_SYMBOL(nr_online_nodes);
  22527. #endif
  22528. +static DEFINE_LOCAL_IRQ_LOCK(pa_lock);
  22529. +
  22530. +#ifdef CONFIG_PREEMPT_RT_BASE
  22531. +# define cpu_lock_irqsave(cpu, flags) \
  22532. + local_lock_irqsave_on(pa_lock, flags, cpu)
  22533. +# define cpu_unlock_irqrestore(cpu, flags) \
  22534. + local_unlock_irqrestore_on(pa_lock, flags, cpu)
  22535. +#else
  22536. +# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags)
  22537. +# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags)
  22538. +#endif
  22539. +
  22540. int page_group_by_mobility_disabled __read_mostly;
  22541. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  22542. @@ -816,7 +829,7 @@
  22543. }
  22544. /*
  22545. - * Frees a number of pages from the PCP lists
  22546. + * Frees a number of pages which have been collected from the pcp lists.
  22547. * Assumes all pages on list are in same zone, and of same order.
  22548. * count is the number of pages to free.
  22549. *
  22550. @@ -827,18 +840,53 @@
  22551. * pinned" detection logic.
  22552. */
  22553. static void free_pcppages_bulk(struct zone *zone, int count,
  22554. - struct per_cpu_pages *pcp)
  22555. + struct list_head *list)
  22556. {
  22557. - int migratetype = 0;
  22558. - int batch_free = 0;
  22559. int to_free = count;
  22560. unsigned long nr_scanned;
  22561. + unsigned long flags;
  22562. +
  22563. + spin_lock_irqsave(&zone->lock, flags);
  22564. - spin_lock(&zone->lock);
  22565. nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
  22566. if (nr_scanned)
  22567. __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
  22568. + while (!list_empty(list)) {
  22569. + struct page *page = list_first_entry(list, struct page, lru);
  22570. + int mt; /* migratetype of the to-be-freed page */
  22571. +
  22572. + /* must delete as __free_one_page list manipulates */
  22573. + list_del(&page->lru);
  22574. +
  22575. + mt = get_pcppage_migratetype(page);
  22576. + /* MIGRATE_ISOLATE page should not go to pcplists */
  22577. + VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
  22578. + /* Pageblock could have been isolated meanwhile */
  22579. + if (unlikely(has_isolate_pageblock(zone)))
  22580. + mt = get_pageblock_migratetype(page);
  22581. +
  22582. + __free_one_page(page, page_to_pfn(page), zone, 0, mt);
  22583. + trace_mm_page_pcpu_drain(page, 0, mt);
  22584. + to_free--;
  22585. + }
  22586. + WARN_ON(to_free != 0);
  22587. + spin_unlock_irqrestore(&zone->lock, flags);
  22588. +}
  22589. +
  22590. +/*
  22591. + * Moves a number of pages from the PCP lists to free list which
  22592. + * is freed outside of the locked region.
  22593. + *
  22594. + * Assumes all pages on list are in same zone, and of same order.
  22595. + * count is the number of pages to free.
  22596. + */
  22597. +static void isolate_pcp_pages(int to_free, struct per_cpu_pages *src,
  22598. + struct list_head *dst)
  22599. +{
  22600. + int migratetype = 0;
  22601. + int batch_free = 0;
  22602. +
  22603. while (to_free) {
  22604. struct page *page;
  22605. struct list_head *list;
  22606. @@ -854,7 +902,7 @@
  22607. batch_free++;
  22608. if (++migratetype == MIGRATE_PCPTYPES)
  22609. migratetype = 0;
  22610. - list = &pcp->lists[migratetype];
  22611. + list = &src->lists[migratetype];
  22612. } while (list_empty(list));
  22613. /* This is the only non-empty list. Free them all. */
  22614. @@ -862,24 +910,12 @@
  22615. batch_free = to_free;
  22616. do {
  22617. - int mt; /* migratetype of the to-be-freed page */
  22618. -
  22619. page = list_last_entry(list, struct page, lru);
  22620. - /* must delete as __free_one_page list manipulates */
  22621. list_del(&page->lru);
  22622. - mt = get_pcppage_migratetype(page);
  22623. - /* MIGRATE_ISOLATE page should not go to pcplists */
  22624. - VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
  22625. - /* Pageblock could have been isolated meanwhile */
  22626. - if (unlikely(has_isolate_pageblock(zone)))
  22627. - mt = get_pageblock_migratetype(page);
  22628. -
  22629. - __free_one_page(page, page_to_pfn(page), zone, 0, mt);
  22630. - trace_mm_page_pcpu_drain(page, 0, mt);
  22631. + list_add(&page->lru, dst);
  22632. } while (--to_free && --batch_free && !list_empty(list));
  22633. }
  22634. - spin_unlock(&zone->lock);
  22635. }
  22636. static void free_one_page(struct zone *zone,
  22637. @@ -888,7 +924,9 @@
  22638. int migratetype)
  22639. {
  22640. unsigned long nr_scanned;
  22641. - spin_lock(&zone->lock);
  22642. + unsigned long flags;
  22643. +
  22644. + spin_lock_irqsave(&zone->lock, flags);
  22645. nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
  22646. if (nr_scanned)
  22647. __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
  22648. @@ -898,7 +936,7 @@
  22649. migratetype = get_pfnblock_migratetype(page, pfn);
  22650. }
  22651. __free_one_page(page, pfn, zone, order, migratetype);
  22652. - spin_unlock(&zone->lock);
  22653. + spin_unlock_irqrestore(&zone->lock, flags);
  22654. }
  22655. static int free_tail_pages_check(struct page *head_page, struct page *page)
  22656. @@ -1072,10 +1110,10 @@
  22657. return;
  22658. migratetype = get_pfnblock_migratetype(page, pfn);
  22659. - local_irq_save(flags);
  22660. + local_lock_irqsave(pa_lock, flags);
  22661. __count_vm_events(PGFREE, 1 << order);
  22662. free_one_page(page_zone(page), page, pfn, order, migratetype);
  22663. - local_irq_restore(flags);
  22664. + local_unlock_irqrestore(pa_lock, flags);
  22665. }
  22666. static void __init __free_pages_boot_core(struct page *page,
  22667. @@ -2017,16 +2055,18 @@
  22668. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  22669. {
  22670. unsigned long flags;
  22671. + LIST_HEAD(dst);
  22672. int to_drain, batch;
  22673. - local_irq_save(flags);
  22674. + local_lock_irqsave(pa_lock, flags);
  22675. batch = READ_ONCE(pcp->batch);
  22676. to_drain = min(pcp->count, batch);
  22677. if (to_drain > 0) {
  22678. - free_pcppages_bulk(zone, to_drain, pcp);
  22679. + isolate_pcp_pages(to_drain, pcp, &dst);
  22680. pcp->count -= to_drain;
  22681. }
  22682. - local_irq_restore(flags);
  22683. + local_unlock_irqrestore(pa_lock, flags);
  22684. + free_pcppages_bulk(zone, to_drain, &dst);
  22685. }
  22686. #endif
  22687. @@ -2042,16 +2082,21 @@
  22688. unsigned long flags;
  22689. struct per_cpu_pageset *pset;
  22690. struct per_cpu_pages *pcp;
  22691. + LIST_HEAD(dst);
  22692. + int count;
  22693. - local_irq_save(flags);
  22694. + cpu_lock_irqsave(cpu, flags);
  22695. pset = per_cpu_ptr(zone->pageset, cpu);
  22696. pcp = &pset->pcp;
  22697. - if (pcp->count) {
  22698. - free_pcppages_bulk(zone, pcp->count, pcp);
  22699. + count = pcp->count;
  22700. + if (count) {
  22701. + isolate_pcp_pages(count, pcp, &dst);
  22702. pcp->count = 0;
  22703. }
  22704. - local_irq_restore(flags);
  22705. + cpu_unlock_irqrestore(cpu, flags);
  22706. + if (count)
  22707. + free_pcppages_bulk(zone, count, &dst);
  22708. }
  22709. /*
  22710. @@ -2137,8 +2182,17 @@
  22711. else
  22712. cpumask_clear_cpu(cpu, &cpus_with_pcps);
  22713. }
  22714. +#ifndef CONFIG_PREEMPT_RT_BASE
  22715. on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
  22716. zone, 1);
  22717. +#else
  22718. + for_each_cpu(cpu, &cpus_with_pcps) {
  22719. + if (zone)
  22720. + drain_pages_zone(cpu, zone);
  22721. + else
  22722. + drain_pages(cpu);
  22723. + }
  22724. +#endif
  22725. }
  22726. #ifdef CONFIG_HIBERNATION
  22727. @@ -2194,7 +2248,7 @@
  22728. migratetype = get_pfnblock_migratetype(page, pfn);
  22729. set_pcppage_migratetype(page, migratetype);
  22730. - local_irq_save(flags);
  22731. + local_lock_irqsave(pa_lock, flags);
  22732. __count_vm_event(PGFREE);
  22733. /*
  22734. @@ -2220,12 +2274,17 @@
  22735. pcp->count++;
  22736. if (pcp->count >= pcp->high) {
  22737. unsigned long batch = READ_ONCE(pcp->batch);
  22738. - free_pcppages_bulk(zone, batch, pcp);
  22739. + LIST_HEAD(dst);
  22740. +
  22741. + isolate_pcp_pages(batch, pcp, &dst);
  22742. pcp->count -= batch;
  22743. + local_unlock_irqrestore(pa_lock, flags);
  22744. + free_pcppages_bulk(zone, batch, &dst);
  22745. + return;
  22746. }
  22747. out:
  22748. - local_irq_restore(flags);
  22749. + local_unlock_irqrestore(pa_lock, flags);
  22750. }
  22751. /*
  22752. @@ -2360,7 +2419,7 @@
  22753. struct per_cpu_pages *pcp;
  22754. struct list_head *list;
  22755. - local_irq_save(flags);
  22756. + local_lock_irqsave(pa_lock, flags);
  22757. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  22758. list = &pcp->lists[migratetype];
  22759. if (list_empty(list)) {
  22760. @@ -2384,7 +2443,7 @@
  22761. * allocate greater than order-1 page units with __GFP_NOFAIL.
  22762. */
  22763. WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
  22764. - spin_lock_irqsave(&zone->lock, flags);
  22765. + local_spin_lock_irqsave(pa_lock, &zone->lock, flags);
  22766. page = NULL;
  22767. if (alloc_flags & ALLOC_HARDER) {
  22768. @@ -2394,11 +2453,13 @@
  22769. }
  22770. if (!page)
  22771. page = __rmqueue(zone, order, migratetype);
  22772. - spin_unlock(&zone->lock);
  22773. - if (!page)
  22774. + if (!page) {
  22775. + spin_unlock(&zone->lock);
  22776. goto failed;
  22777. + }
  22778. __mod_zone_freepage_state(zone, -(1 << order),
  22779. get_pcppage_migratetype(page));
  22780. + spin_unlock(&zone->lock);
  22781. }
  22782. __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
  22783. @@ -2408,13 +2469,13 @@
  22784. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  22785. zone_statistics(preferred_zone, zone, gfp_flags);
  22786. - local_irq_restore(flags);
  22787. + local_unlock_irqrestore(pa_lock, flags);
  22788. VM_BUG_ON_PAGE(bad_range(zone, page), page);
  22789. return page;
  22790. failed:
  22791. - local_irq_restore(flags);
  22792. + local_unlock_irqrestore(pa_lock, flags);
  22793. return NULL;
  22794. }
  22795. @@ -6215,7 +6276,9 @@
  22796. int cpu = (unsigned long)hcpu;
  22797. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  22798. + local_lock_irq_on(swapvec_lock, cpu);
  22799. lru_add_drain_cpu(cpu);
  22800. + local_unlock_irq_on(swapvec_lock, cpu);
  22801. drain_pages(cpu);
  22802. /*
  22803. @@ -6241,6 +6304,7 @@
  22804. void __init page_alloc_init(void)
  22805. {
  22806. hotcpu_notifier(page_alloc_cpu_notify, 0);
  22807. + local_irq_lock_init(pa_lock);
  22808. }
  22809. /*
  22810. @@ -7165,7 +7229,7 @@
  22811. struct per_cpu_pageset *pset;
  22812. /* avoid races with drain_pages() */
  22813. - local_irq_save(flags);
  22814. + local_lock_irqsave(pa_lock, flags);
  22815. if (zone->pageset != &boot_pageset) {
  22816. for_each_online_cpu(cpu) {
  22817. pset = per_cpu_ptr(zone->pageset, cpu);
  22818. @@ -7174,7 +7238,7 @@
  22819. free_percpu(zone->pageset);
  22820. zone->pageset = &boot_pageset;
  22821. }
  22822. - local_irq_restore(flags);
  22823. + local_unlock_irqrestore(pa_lock, flags);
  22824. }
  22825. #ifdef CONFIG_MEMORY_HOTREMOVE
  22826. diff -Nur linux-4.6.6.orig/mm/slab.h linux-4.6.6/mm/slab.h
  22827. --- linux-4.6.6.orig/mm/slab.h 2016-08-10 12:54:57.000000000 +0200
  22828. +++ linux-4.6.6/mm/slab.h 2016-08-22 21:27:47.441894907 +0200
  22829. @@ -415,7 +415,11 @@
  22830. * The slab lists for all objects.
  22831. */
  22832. struct kmem_cache_node {
  22833. +#ifdef CONFIG_SLUB
  22834. + raw_spinlock_t list_lock;
  22835. +#else
  22836. spinlock_t list_lock;
  22837. +#endif
  22838. #ifdef CONFIG_SLAB
  22839. struct list_head slabs_partial; /* partial list first, better asm code */
  22840. diff -Nur linux-4.6.6.orig/mm/slub.c linux-4.6.6/mm/slub.c
  22841. --- linux-4.6.6.orig/mm/slub.c 2016-08-10 12:54:57.000000000 +0200
  22842. +++ linux-4.6.6/mm/slub.c 2016-08-22 21:27:47.445895167 +0200
  22843. @@ -1143,7 +1143,7 @@
  22844. unsigned long uninitialized_var(flags);
  22845. int ret = 0;
  22846. - spin_lock_irqsave(&n->list_lock, flags);
  22847. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22848. slab_lock(page);
  22849. if (s->flags & SLAB_CONSISTENCY_CHECKS) {
  22850. @@ -1178,7 +1178,7 @@
  22851. bulk_cnt, cnt);
  22852. slab_unlock(page);
  22853. - spin_unlock_irqrestore(&n->list_lock, flags);
  22854. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22855. if (!ret)
  22856. slab_fix(s, "Object at 0x%p not freed", object);
  22857. return ret;
  22858. @@ -1306,6 +1306,12 @@
  22859. #endif /* CONFIG_SLUB_DEBUG */
  22860. +struct slub_free_list {
  22861. + raw_spinlock_t lock;
  22862. + struct list_head list;
  22863. +};
  22864. +static DEFINE_PER_CPU(struct slub_free_list, slub_free_list);
  22865. +
  22866. /*
  22867. * Hooks for other subsystems that check memory allocations. In a typical
  22868. * production configuration these hooks all should produce no code at all.
  22869. @@ -1412,10 +1418,17 @@
  22870. gfp_t alloc_gfp;
  22871. void *start, *p;
  22872. int idx, order;
  22873. + bool enableirqs = false;
  22874. flags &= gfp_allowed_mask;
  22875. if (gfpflags_allow_blocking(flags))
  22876. + enableirqs = true;
  22877. +#ifdef CONFIG_PREEMPT_RT_FULL
  22878. + if (system_state == SYSTEM_RUNNING)
  22879. + enableirqs = true;
  22880. +#endif
  22881. + if (enableirqs)
  22882. local_irq_enable();
  22883. flags |= s->allocflags;
  22884. @@ -1486,7 +1499,7 @@
  22885. page->frozen = 1;
  22886. out:
  22887. - if (gfpflags_allow_blocking(flags))
  22888. + if (enableirqs)
  22889. local_irq_disable();
  22890. if (!page)
  22891. return NULL;
  22892. @@ -1543,6 +1556,16 @@
  22893. __free_pages(page, order);
  22894. }
  22895. +static void free_delayed(struct list_head *h)
  22896. +{
  22897. + while(!list_empty(h)) {
  22898. + struct page *page = list_first_entry(h, struct page, lru);
  22899. +
  22900. + list_del(&page->lru);
  22901. + __free_slab(page->slab_cache, page);
  22902. + }
  22903. +}
  22904. +
  22905. #define need_reserve_slab_rcu \
  22906. (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
  22907. @@ -1574,6 +1597,12 @@
  22908. }
  22909. call_rcu(head, rcu_free_slab);
  22910. + } else if (irqs_disabled()) {
  22911. + struct slub_free_list *f = this_cpu_ptr(&slub_free_list);
  22912. +
  22913. + raw_spin_lock(&f->lock);
  22914. + list_add(&page->lru, &f->list);
  22915. + raw_spin_unlock(&f->lock);
  22916. } else
  22917. __free_slab(s, page);
  22918. }
  22919. @@ -1681,7 +1710,7 @@
  22920. if (!n || !n->nr_partial)
  22921. return NULL;
  22922. - spin_lock(&n->list_lock);
  22923. + raw_spin_lock(&n->list_lock);
  22924. list_for_each_entry_safe(page, page2, &n->partial, lru) {
  22925. void *t;
  22926. @@ -1706,7 +1735,7 @@
  22927. break;
  22928. }
  22929. - spin_unlock(&n->list_lock);
  22930. + raw_spin_unlock(&n->list_lock);
  22931. return object;
  22932. }
  22933. @@ -1952,7 +1981,7 @@
  22934. * that acquire_slab() will see a slab page that
  22935. * is frozen
  22936. */
  22937. - spin_lock(&n->list_lock);
  22938. + raw_spin_lock(&n->list_lock);
  22939. }
  22940. } else {
  22941. m = M_FULL;
  22942. @@ -1963,7 +1992,7 @@
  22943. * slabs from diagnostic functions will not see
  22944. * any frozen slabs.
  22945. */
  22946. - spin_lock(&n->list_lock);
  22947. + raw_spin_lock(&n->list_lock);
  22948. }
  22949. }
  22950. @@ -1998,7 +2027,7 @@
  22951. goto redo;
  22952. if (lock)
  22953. - spin_unlock(&n->list_lock);
  22954. + raw_spin_unlock(&n->list_lock);
  22955. if (m == M_FREE) {
  22956. stat(s, DEACTIVATE_EMPTY);
  22957. @@ -2030,10 +2059,10 @@
  22958. n2 = get_node(s, page_to_nid(page));
  22959. if (n != n2) {
  22960. if (n)
  22961. - spin_unlock(&n->list_lock);
  22962. + raw_spin_unlock(&n->list_lock);
  22963. n = n2;
  22964. - spin_lock(&n->list_lock);
  22965. + raw_spin_lock(&n->list_lock);
  22966. }
  22967. do {
  22968. @@ -2062,7 +2091,7 @@
  22969. }
  22970. if (n)
  22971. - spin_unlock(&n->list_lock);
  22972. + raw_spin_unlock(&n->list_lock);
  22973. while (discard_page) {
  22974. page = discard_page;
  22975. @@ -2101,14 +2130,21 @@
  22976. pobjects = oldpage->pobjects;
  22977. pages = oldpage->pages;
  22978. if (drain && pobjects > s->cpu_partial) {
  22979. + struct slub_free_list *f;
  22980. unsigned long flags;
  22981. + LIST_HEAD(tofree);
  22982. /*
  22983. * partial array is full. Move the existing
  22984. * set to the per node partial list.
  22985. */
  22986. local_irq_save(flags);
  22987. unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
  22988. + f = this_cpu_ptr(&slub_free_list);
  22989. + raw_spin_lock(&f->lock);
  22990. + list_splice_init(&f->list, &tofree);
  22991. + raw_spin_unlock(&f->lock);
  22992. local_irq_restore(flags);
  22993. + free_delayed(&tofree);
  22994. oldpage = NULL;
  22995. pobjects = 0;
  22996. pages = 0;
  22997. @@ -2180,7 +2216,22 @@
  22998. static void flush_all(struct kmem_cache *s)
  22999. {
  23000. + LIST_HEAD(tofree);
  23001. + int cpu;
  23002. +
  23003. on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
  23004. + for_each_online_cpu(cpu) {
  23005. + struct slub_free_list *f;
  23006. +
  23007. + if (!has_cpu_slab(cpu, s))
  23008. + continue;
  23009. +
  23010. + f = &per_cpu(slub_free_list, cpu);
  23011. + raw_spin_lock_irq(&f->lock);
  23012. + list_splice_init(&f->list, &tofree);
  23013. + raw_spin_unlock_irq(&f->lock);
  23014. + free_delayed(&tofree);
  23015. + }
  23016. }
  23017. /*
  23018. @@ -2216,10 +2267,10 @@
  23019. unsigned long x = 0;
  23020. struct page *page;
  23021. - spin_lock_irqsave(&n->list_lock, flags);
  23022. + raw_spin_lock_irqsave(&n->list_lock, flags);
  23023. list_for_each_entry(page, &n->partial, lru)
  23024. x += get_count(page);
  23025. - spin_unlock_irqrestore(&n->list_lock, flags);
  23026. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23027. return x;
  23028. }
  23029. #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
  23030. @@ -2357,8 +2408,10 @@
  23031. * already disabled (which is the case for bulk allocation).
  23032. */
  23033. static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
  23034. - unsigned long addr, struct kmem_cache_cpu *c)
  23035. + unsigned long addr, struct kmem_cache_cpu *c,
  23036. + struct list_head *to_free)
  23037. {
  23038. + struct slub_free_list *f;
  23039. void *freelist;
  23040. struct page *page;
  23041. @@ -2418,6 +2471,13 @@
  23042. VM_BUG_ON(!c->page->frozen);
  23043. c->freelist = get_freepointer(s, freelist);
  23044. c->tid = next_tid(c->tid);
  23045. +
  23046. +out:
  23047. + f = this_cpu_ptr(&slub_free_list);
  23048. + raw_spin_lock(&f->lock);
  23049. + list_splice_init(&f->list, to_free);
  23050. + raw_spin_unlock(&f->lock);
  23051. +
  23052. return freelist;
  23053. new_slab:
  23054. @@ -2449,7 +2509,7 @@
  23055. deactivate_slab(s, page, get_freepointer(s, freelist));
  23056. c->page = NULL;
  23057. c->freelist = NULL;
  23058. - return freelist;
  23059. + goto out;
  23060. }
  23061. /*
  23062. @@ -2461,6 +2521,7 @@
  23063. {
  23064. void *p;
  23065. unsigned long flags;
  23066. + LIST_HEAD(tofree);
  23067. local_irq_save(flags);
  23068. #ifdef CONFIG_PREEMPT
  23069. @@ -2472,8 +2533,9 @@
  23070. c = this_cpu_ptr(s->cpu_slab);
  23071. #endif
  23072. - p = ___slab_alloc(s, gfpflags, node, addr, c);
  23073. + p = ___slab_alloc(s, gfpflags, node, addr, c, &tofree);
  23074. local_irq_restore(flags);
  23075. + free_delayed(&tofree);
  23076. return p;
  23077. }
  23078. @@ -2659,7 +2721,7 @@
  23079. do {
  23080. if (unlikely(n)) {
  23081. - spin_unlock_irqrestore(&n->list_lock, flags);
  23082. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23083. n = NULL;
  23084. }
  23085. prior = page->freelist;
  23086. @@ -2691,7 +2753,7 @@
  23087. * Otherwise the list_lock will synchronize with
  23088. * other processors updating the list of slabs.
  23089. */
  23090. - spin_lock_irqsave(&n->list_lock, flags);
  23091. + raw_spin_lock_irqsave(&n->list_lock, flags);
  23092. }
  23093. }
  23094. @@ -2733,7 +2795,7 @@
  23095. add_partial(n, page, DEACTIVATE_TO_TAIL);
  23096. stat(s, FREE_ADD_PARTIAL);
  23097. }
  23098. - spin_unlock_irqrestore(&n->list_lock, flags);
  23099. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23100. return;
  23101. slab_empty:
  23102. @@ -2748,7 +2810,7 @@
  23103. remove_full(s, n, page);
  23104. }
  23105. - spin_unlock_irqrestore(&n->list_lock, flags);
  23106. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23107. stat(s, FREE_SLAB);
  23108. discard_slab(s, page);
  23109. }
  23110. @@ -2935,6 +2997,7 @@
  23111. void **p)
  23112. {
  23113. struct kmem_cache_cpu *c;
  23114. + LIST_HEAD(to_free);
  23115. int i;
  23116. /* memcg and kmem_cache debug support */
  23117. @@ -2958,7 +3021,7 @@
  23118. * of re-populating per CPU c->freelist
  23119. */
  23120. p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
  23121. - _RET_IP_, c);
  23122. + _RET_IP_, c, &to_free);
  23123. if (unlikely(!p[i]))
  23124. goto error;
  23125. @@ -2970,6 +3033,7 @@
  23126. }
  23127. c->tid = next_tid(c->tid);
  23128. local_irq_enable();
  23129. + free_delayed(&to_free);
  23130. /* Clear memory outside IRQ disabled fastpath loop */
  23131. if (unlikely(flags & __GFP_ZERO)) {
  23132. @@ -3117,7 +3181,7 @@
  23133. init_kmem_cache_node(struct kmem_cache_node *n)
  23134. {
  23135. n->nr_partial = 0;
  23136. - spin_lock_init(&n->list_lock);
  23137. + raw_spin_lock_init(&n->list_lock);
  23138. INIT_LIST_HEAD(&n->partial);
  23139. #ifdef CONFIG_SLUB_DEBUG
  23140. atomic_long_set(&n->nr_slabs, 0);
  23141. @@ -3450,6 +3514,10 @@
  23142. const char *text)
  23143. {
  23144. #ifdef CONFIG_SLUB_DEBUG
  23145. +#ifdef CONFIG_PREEMPT_RT_BASE
  23146. + /* XXX move out of irq-off section */
  23147. + slab_err(s, page, text, s->name);
  23148. +#else
  23149. void *addr = page_address(page);
  23150. void *p;
  23151. unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
  23152. @@ -3470,6 +3538,7 @@
  23153. slab_unlock(page);
  23154. kfree(map);
  23155. #endif
  23156. +#endif
  23157. }
  23158. /*
  23159. @@ -3482,7 +3551,7 @@
  23160. struct page *page, *h;
  23161. BUG_ON(irqs_disabled());
  23162. - spin_lock_irq(&n->list_lock);
  23163. + raw_spin_lock_irq(&n->list_lock);
  23164. list_for_each_entry_safe(page, h, &n->partial, lru) {
  23165. if (!page->inuse) {
  23166. remove_partial(n, page);
  23167. @@ -3492,7 +3561,7 @@
  23168. "Objects remaining in %s on __kmem_cache_shutdown()");
  23169. }
  23170. }
  23171. - spin_unlock_irq(&n->list_lock);
  23172. + raw_spin_unlock_irq(&n->list_lock);
  23173. }
  23174. /*
  23175. @@ -3706,7 +3775,7 @@
  23176. for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
  23177. INIT_LIST_HEAD(promote + i);
  23178. - spin_lock_irqsave(&n->list_lock, flags);
  23179. + raw_spin_lock_irqsave(&n->list_lock, flags);
  23180. /*
  23181. * Build lists of slabs to discard or promote.
  23182. @@ -3737,7 +3806,7 @@
  23183. for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
  23184. list_splice(promote + i, &n->partial);
  23185. - spin_unlock_irqrestore(&n->list_lock, flags);
  23186. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23187. /* Release empty slabs */
  23188. list_for_each_entry_safe(page, t, &discard, lru)
  23189. @@ -3913,6 +3982,12 @@
  23190. {
  23191. static __initdata struct kmem_cache boot_kmem_cache,
  23192. boot_kmem_cache_node;
  23193. + int cpu;
  23194. +
  23195. + for_each_possible_cpu(cpu) {
  23196. + raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock);
  23197. + INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list);
  23198. + }
  23199. if (debug_guardpage_minorder())
  23200. slub_max_order = 0;
  23201. @@ -4156,7 +4231,7 @@
  23202. struct page *page;
  23203. unsigned long flags;
  23204. - spin_lock_irqsave(&n->list_lock, flags);
  23205. + raw_spin_lock_irqsave(&n->list_lock, flags);
  23206. list_for_each_entry(page, &n->partial, lru) {
  23207. validate_slab_slab(s, page, map);
  23208. @@ -4178,7 +4253,7 @@
  23209. s->name, count, atomic_long_read(&n->nr_slabs));
  23210. out:
  23211. - spin_unlock_irqrestore(&n->list_lock, flags);
  23212. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23213. return count;
  23214. }
  23215. @@ -4366,12 +4441,12 @@
  23216. if (!atomic_long_read(&n->nr_slabs))
  23217. continue;
  23218. - spin_lock_irqsave(&n->list_lock, flags);
  23219. + raw_spin_lock_irqsave(&n->list_lock, flags);
  23220. list_for_each_entry(page, &n->partial, lru)
  23221. process_slab(&t, s, page, alloc, map);
  23222. list_for_each_entry(page, &n->full, lru)
  23223. process_slab(&t, s, page, alloc, map);
  23224. - spin_unlock_irqrestore(&n->list_lock, flags);
  23225. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  23226. }
  23227. for (i = 0; i < t.count; i++) {
  23228. diff -Nur linux-4.6.6.orig/mm/swap.c linux-4.6.6/mm/swap.c
  23229. --- linux-4.6.6.orig/mm/swap.c 2016-08-10 12:54:57.000000000 +0200
  23230. +++ linux-4.6.6/mm/swap.c 2016-08-22 21:37:54.852846351 +0200
  23231. @@ -32,6 +32,7 @@
  23232. #include <linux/memcontrol.h>
  23233. #include <linux/gfp.h>
  23234. #include <linux/uio.h>
  23235. +#include <linux/locallock.h>
  23236. #include <linux/hugetlb.h>
  23237. #include <linux/page_idle.h>
  23238. @@ -48,6 +49,9 @@
  23239. static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
  23240. static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
  23241. +static DEFINE_LOCAL_IRQ_LOCK(rotate_lock);
  23242. +DEFINE_LOCAL_IRQ_LOCK(swapvec_lock);
  23243. +
  23244. /*
  23245. * This path almost never happens for VM activity - pages are normally
  23246. * freed via pagevecs. But it gets used by networking.
  23247. @@ -237,11 +241,11 @@
  23248. unsigned long flags;
  23249. get_page(page);
  23250. - local_irq_save(flags);
  23251. + local_lock_irqsave(rotate_lock, flags);
  23252. pvec = this_cpu_ptr(&lru_rotate_pvecs);
  23253. if (!pagevec_add(pvec, page) || PageCompound(page))
  23254. pagevec_move_tail(pvec);
  23255. - local_irq_restore(flags);
  23256. + local_unlock_irqrestore(rotate_lock, flags);
  23257. }
  23258. }
  23259. @@ -292,12 +296,13 @@
  23260. void activate_page(struct page *page)
  23261. {
  23262. if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
  23263. - struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
  23264. + struct pagevec *pvec = &get_locked_var(swapvec_lock,
  23265. + activate_page_pvecs);
  23266. get_page(page);
  23267. if (!pagevec_add(pvec, page) || PageCompound(page))
  23268. pagevec_lru_move_fn(pvec, __activate_page, NULL);
  23269. - put_cpu_var(activate_page_pvecs);
  23270. + put_locked_var(swapvec_lock, activate_page_pvecs);
  23271. }
  23272. }
  23273. @@ -323,7 +328,7 @@
  23274. static void __lru_cache_activate_page(struct page *page)
  23275. {
  23276. - struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
  23277. + struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec);
  23278. int i;
  23279. /*
  23280. @@ -345,7 +350,7 @@
  23281. }
  23282. }
  23283. - put_cpu_var(lru_add_pvec);
  23284. + put_locked_var(swapvec_lock, lru_add_pvec);
  23285. }
  23286. /*
  23287. @@ -387,12 +392,12 @@
  23288. static void __lru_cache_add(struct page *page)
  23289. {
  23290. - struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
  23291. + struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec);
  23292. get_page(page);
  23293. if (!pagevec_add(pvec, page) || PageCompound(page))
  23294. __pagevec_lru_add(pvec);
  23295. - put_cpu_var(lru_add_pvec);
  23296. + put_locked_var(swapvec_lock, lru_add_pvec);
  23297. }
  23298. /**
  23299. @@ -590,9 +595,15 @@
  23300. unsigned long flags;
  23301. /* No harm done if a racing interrupt already did this */
  23302. - local_irq_save(flags);
  23303. +#ifdef CONFIG_PREEMPT_RT_BASE
  23304. + local_lock_irqsave_on(rotate_lock, flags, cpu);
  23305. + pagevec_move_tail(pvec);
  23306. + local_unlock_irqrestore_on(rotate_lock, flags, cpu);
  23307. +#else
  23308. + local_lock_irqsave(rotate_lock, flags);
  23309. pagevec_move_tail(pvec);
  23310. - local_irq_restore(flags);
  23311. + local_unlock_irqrestore(rotate_lock, flags);
  23312. +#endif
  23313. }
  23314. pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
  23315. @@ -643,27 +654,47 @@
  23316. void deactivate_page(struct page *page)
  23317. {
  23318. if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
  23319. - struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
  23320. + struct pagevec *pvec = &get_locked_var(swapvec_lock,
  23321. + lru_deactivate_file_pvecs);
  23322. get_page(page);
  23323. if (!pagevec_add(pvec, page) || PageCompound(page))
  23324. pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
  23325. - put_cpu_var(lru_deactivate_pvecs);
  23326. + put_locked_var(swapvec_lock, lru_deactivate_file_pvecs);
  23327. }
  23328. }
  23329. void lru_add_drain(void)
  23330. {
  23331. - lru_add_drain_cpu(get_cpu());
  23332. - put_cpu();
  23333. + lru_add_drain_cpu(local_lock_cpu(swapvec_lock));
  23334. + local_unlock_cpu(swapvec_lock);
  23335. }
  23336. +#ifdef CONFIG_PREEMPT_RT_BASE
  23337. +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
  23338. +{
  23339. + local_lock_on(swapvec_lock, cpu);
  23340. + lru_add_drain_cpu(cpu);
  23341. + local_unlock_on(swapvec_lock, cpu);
  23342. +}
  23343. +
  23344. +#else
  23345. +
  23346. static void lru_add_drain_per_cpu(struct work_struct *dummy)
  23347. {
  23348. lru_add_drain();
  23349. }
  23350. static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
  23351. +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
  23352. +{
  23353. + struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
  23354. +
  23355. + INIT_WORK(work, lru_add_drain_per_cpu);
  23356. + schedule_work_on(cpu, work);
  23357. + cpumask_set_cpu(cpu, has_work);
  23358. +}
  23359. +#endif
  23360. void lru_add_drain_all(void)
  23361. {
  23362. @@ -676,21 +707,18 @@
  23363. cpumask_clear(&has_work);
  23364. for_each_online_cpu(cpu) {
  23365. - struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
  23366. -
  23367. if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
  23368. pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
  23369. pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
  23370. pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
  23371. - need_activate_page_drain(cpu)) {
  23372. - INIT_WORK(work, lru_add_drain_per_cpu);
  23373. - schedule_work_on(cpu, work);
  23374. - cpumask_set_cpu(cpu, &has_work);
  23375. - }
  23376. + need_activate_page_drain(cpu))
  23377. + remote_lru_add_drain(cpu, &has_work);
  23378. }
  23379. +#ifndef CONFIG_PREEMPT_RT_BASE
  23380. for_each_cpu(cpu, &has_work)
  23381. flush_work(&per_cpu(lru_add_drain_work, cpu));
  23382. +#endif
  23383. put_online_cpus();
  23384. mutex_unlock(&lock);
  23385. diff -Nur linux-4.6.6.orig/mm/truncate.c linux-4.6.6/mm/truncate.c
  23386. --- linux-4.6.6.orig/mm/truncate.c 2016-08-10 12:54:57.000000000 +0200
  23387. +++ linux-4.6.6/mm/truncate.c 2016-08-22 21:27:47.445895167 +0200
  23388. @@ -63,9 +63,12 @@
  23389. * protected by mapping->tree_lock.
  23390. */
  23391. if (!workingset_node_shadows(node) &&
  23392. - !list_empty(&node->private_list))
  23393. - list_lru_del(&workingset_shadow_nodes,
  23394. + !list_empty(&node->private_list)) {
  23395. + local_lock(workingset_shadow_lock);
  23396. + list_lru_del(&__workingset_shadow_nodes,
  23397. &node->private_list);
  23398. + local_unlock(workingset_shadow_lock);
  23399. + }
  23400. __radix_tree_delete_node(&mapping->page_tree, node);
  23401. }
  23402. unlock:
  23403. diff -Nur linux-4.6.6.orig/mm/vmalloc.c linux-4.6.6/mm/vmalloc.c
  23404. --- linux-4.6.6.orig/mm/vmalloc.c 2016-08-10 12:54:57.000000000 +0200
  23405. +++ linux-4.6.6/mm/vmalloc.c 2016-08-22 21:27:47.445895167 +0200
  23406. @@ -819,7 +819,7 @@
  23407. struct vmap_block *vb;
  23408. struct vmap_area *va;
  23409. unsigned long vb_idx;
  23410. - int node, err;
  23411. + int node, err, cpu;
  23412. void *vaddr;
  23413. node = numa_node_id();
  23414. @@ -862,11 +862,12 @@
  23415. BUG_ON(err);
  23416. radix_tree_preload_end();
  23417. - vbq = &get_cpu_var(vmap_block_queue);
  23418. + cpu = get_cpu_light();
  23419. + vbq = this_cpu_ptr(&vmap_block_queue);
  23420. spin_lock(&vbq->lock);
  23421. list_add_tail_rcu(&vb->free_list, &vbq->free);
  23422. spin_unlock(&vbq->lock);
  23423. - put_cpu_var(vmap_block_queue);
  23424. + put_cpu_light();
  23425. return vaddr;
  23426. }
  23427. @@ -935,6 +936,7 @@
  23428. struct vmap_block *vb;
  23429. void *vaddr = NULL;
  23430. unsigned int order;
  23431. + int cpu;
  23432. BUG_ON(offset_in_page(size));
  23433. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  23434. @@ -949,7 +951,8 @@
  23435. order = get_order(size);
  23436. rcu_read_lock();
  23437. - vbq = &get_cpu_var(vmap_block_queue);
  23438. + cpu = get_cpu_light();
  23439. + vbq = this_cpu_ptr(&vmap_block_queue);
  23440. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  23441. unsigned long pages_off;
  23442. @@ -972,7 +975,7 @@
  23443. break;
  23444. }
  23445. - put_cpu_var(vmap_block_queue);
  23446. + put_cpu_light();
  23447. rcu_read_unlock();
  23448. /* Allocate new block if nothing was found */
  23449. diff -Nur linux-4.6.6.orig/mm/vmstat.c linux-4.6.6/mm/vmstat.c
  23450. --- linux-4.6.6.orig/mm/vmstat.c 2016-08-10 12:54:57.000000000 +0200
  23451. +++ linux-4.6.6/mm/vmstat.c 2016-08-22 21:27:47.445895167 +0200
  23452. @@ -226,6 +226,7 @@
  23453. long x;
  23454. long t;
  23455. + preempt_disable_rt();
  23456. x = delta + __this_cpu_read(*p);
  23457. t = __this_cpu_read(pcp->stat_threshold);
  23458. @@ -235,6 +236,7 @@
  23459. x = 0;
  23460. }
  23461. __this_cpu_write(*p, x);
  23462. + preempt_enable_rt();
  23463. }
  23464. EXPORT_SYMBOL(__mod_zone_page_state);
  23465. @@ -267,6 +269,7 @@
  23466. s8 __percpu *p = pcp->vm_stat_diff + item;
  23467. s8 v, t;
  23468. + preempt_disable_rt();
  23469. v = __this_cpu_inc_return(*p);
  23470. t = __this_cpu_read(pcp->stat_threshold);
  23471. if (unlikely(v > t)) {
  23472. @@ -275,6 +278,7 @@
  23473. zone_page_state_add(v + overstep, zone, item);
  23474. __this_cpu_write(*p, -overstep);
  23475. }
  23476. + preempt_enable_rt();
  23477. }
  23478. void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
  23479. @@ -289,6 +293,7 @@
  23480. s8 __percpu *p = pcp->vm_stat_diff + item;
  23481. s8 v, t;
  23482. + preempt_disable_rt();
  23483. v = __this_cpu_dec_return(*p);
  23484. t = __this_cpu_read(pcp->stat_threshold);
  23485. if (unlikely(v < - t)) {
  23486. @@ -297,6 +302,7 @@
  23487. zone_page_state_add(v - overstep, zone, item);
  23488. __this_cpu_write(*p, overstep);
  23489. }
  23490. + preempt_enable_rt();
  23491. }
  23492. void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
  23493. diff -Nur linux-4.6.6.orig/mm/workingset.c linux-4.6.6/mm/workingset.c
  23494. --- linux-4.6.6.orig/mm/workingset.c 2016-08-10 12:54:57.000000000 +0200
  23495. +++ linux-4.6.6/mm/workingset.c 2016-08-22 21:27:47.445895167 +0200
  23496. @@ -335,7 +335,8 @@
  23497. * point where they would still be useful.
  23498. */
  23499. -struct list_lru workingset_shadow_nodes;
  23500. +struct list_lru __workingset_shadow_nodes;
  23501. +DEFINE_LOCAL_IRQ_LOCK(workingset_shadow_lock);
  23502. static unsigned long count_shadow_nodes(struct shrinker *shrinker,
  23503. struct shrink_control *sc)
  23504. @@ -345,9 +346,9 @@
  23505. unsigned long pages;
  23506. /* list_lru lock nests inside IRQ-safe mapping->tree_lock */
  23507. - local_irq_disable();
  23508. - shadow_nodes = list_lru_shrink_count(&workingset_shadow_nodes, sc);
  23509. - local_irq_enable();
  23510. + local_lock_irq(workingset_shadow_lock);
  23511. + shadow_nodes = list_lru_shrink_count(&__workingset_shadow_nodes, sc);
  23512. + local_unlock_irq(workingset_shadow_lock);
  23513. if (memcg_kmem_enabled())
  23514. pages = mem_cgroup_node_nr_lru_pages(sc->memcg, sc->nid,
  23515. @@ -440,9 +441,9 @@
  23516. spin_unlock(&mapping->tree_lock);
  23517. ret = LRU_REMOVED_RETRY;
  23518. out:
  23519. - local_irq_enable();
  23520. + local_unlock_irq(workingset_shadow_lock);
  23521. cond_resched();
  23522. - local_irq_disable();
  23523. + local_lock_irq(workingset_shadow_lock);
  23524. spin_lock(lru_lock);
  23525. return ret;
  23526. }
  23527. @@ -453,10 +454,10 @@
  23528. unsigned long ret;
  23529. /* list_lru lock nests inside IRQ-safe mapping->tree_lock */
  23530. - local_irq_disable();
  23531. - ret = list_lru_shrink_walk(&workingset_shadow_nodes, sc,
  23532. + local_lock_irq(workingset_shadow_lock);
  23533. + ret = list_lru_shrink_walk(&__workingset_shadow_nodes, sc,
  23534. shadow_lru_isolate, NULL);
  23535. - local_irq_enable();
  23536. + local_unlock_irq(workingset_shadow_lock);
  23537. return ret;
  23538. }
  23539. @@ -494,7 +495,7 @@
  23540. printk("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
  23541. timestamp_bits, max_order, bucket_order);
  23542. - ret = list_lru_init_key(&workingset_shadow_nodes, &shadow_nodes_key);
  23543. + ret = list_lru_init_key(&__workingset_shadow_nodes, &shadow_nodes_key);
  23544. if (ret)
  23545. goto err;
  23546. ret = register_shrinker(&workingset_shadow_shrinker);
  23547. @@ -502,7 +503,7 @@
  23548. goto err_list_lru;
  23549. return 0;
  23550. err_list_lru:
  23551. - list_lru_destroy(&workingset_shadow_nodes);
  23552. + list_lru_destroy(&__workingset_shadow_nodes);
  23553. err:
  23554. return ret;
  23555. }
  23556. diff -Nur linux-4.6.6.orig/mm/zsmalloc.c linux-4.6.6/mm/zsmalloc.c
  23557. --- linux-4.6.6.orig/mm/zsmalloc.c 2016-08-10 12:54:57.000000000 +0200
  23558. +++ linux-4.6.6/mm/zsmalloc.c 2016-08-22 21:27:47.449895420 +0200
  23559. @@ -1292,7 +1292,7 @@
  23560. class = pool->size_class[class_idx];
  23561. off = obj_idx_to_offset(page, obj_idx, class->size);
  23562. - area = &get_cpu_var(zs_map_area);
  23563. + area = per_cpu_ptr(&zs_map_area, get_cpu_light());
  23564. area->vm_mm = mm;
  23565. if (off + class->size <= PAGE_SIZE) {
  23566. /* this object is contained entirely within a page */
  23567. @@ -1345,7 +1345,7 @@
  23568. __zs_unmap_object(area, pages, off, class->size);
  23569. }
  23570. - put_cpu_var(zs_map_area);
  23571. + put_cpu_light();
  23572. unpin_tag(handle);
  23573. }
  23574. EXPORT_SYMBOL_GPL(zs_unmap_object);
  23575. diff -Nur linux-4.6.6.orig/net/core/dev.c linux-4.6.6/net/core/dev.c
  23576. --- linux-4.6.6.orig/net/core/dev.c 2016-08-10 12:54:57.000000000 +0200
  23577. +++ linux-4.6.6/net/core/dev.c 2016-08-22 21:27:47.449895420 +0200
  23578. @@ -188,6 +188,7 @@
  23579. static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
  23580. static seqcount_t devnet_rename_seq;
  23581. +static DEFINE_MUTEX(devnet_rename_mutex);
  23582. static inline void dev_base_seq_inc(struct net *net)
  23583. {
  23584. @@ -209,14 +210,14 @@
  23585. static inline void rps_lock(struct softnet_data *sd)
  23586. {
  23587. #ifdef CONFIG_RPS
  23588. - spin_lock(&sd->input_pkt_queue.lock);
  23589. + raw_spin_lock(&sd->input_pkt_queue.raw_lock);
  23590. #endif
  23591. }
  23592. static inline void rps_unlock(struct softnet_data *sd)
  23593. {
  23594. #ifdef CONFIG_RPS
  23595. - spin_unlock(&sd->input_pkt_queue.lock);
  23596. + raw_spin_unlock(&sd->input_pkt_queue.raw_lock);
  23597. #endif
  23598. }
  23599. @@ -886,7 +887,8 @@
  23600. strcpy(name, dev->name);
  23601. rcu_read_unlock();
  23602. if (read_seqcount_retry(&devnet_rename_seq, seq)) {
  23603. - cond_resched();
  23604. + mutex_lock(&devnet_rename_mutex);
  23605. + mutex_unlock(&devnet_rename_mutex);
  23606. goto retry;
  23607. }
  23608. @@ -1155,20 +1157,17 @@
  23609. if (dev->flags & IFF_UP)
  23610. return -EBUSY;
  23611. - write_seqcount_begin(&devnet_rename_seq);
  23612. + mutex_lock(&devnet_rename_mutex);
  23613. + __raw_write_seqcount_begin(&devnet_rename_seq);
  23614. - if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
  23615. - write_seqcount_end(&devnet_rename_seq);
  23616. - return 0;
  23617. - }
  23618. + if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
  23619. + goto outunlock;
  23620. memcpy(oldname, dev->name, IFNAMSIZ);
  23621. err = dev_get_valid_name(net, dev, newname);
  23622. - if (err < 0) {
  23623. - write_seqcount_end(&devnet_rename_seq);
  23624. - return err;
  23625. - }
  23626. + if (err < 0)
  23627. + goto outunlock;
  23628. if (oldname[0] && !strchr(oldname, '%'))
  23629. netdev_info(dev, "renamed from %s\n", oldname);
  23630. @@ -1181,11 +1180,12 @@
  23631. if (ret) {
  23632. memcpy(dev->name, oldname, IFNAMSIZ);
  23633. dev->name_assign_type = old_assign_type;
  23634. - write_seqcount_end(&devnet_rename_seq);
  23635. - return ret;
  23636. + err = ret;
  23637. + goto outunlock;
  23638. }
  23639. - write_seqcount_end(&devnet_rename_seq);
  23640. + __raw_write_seqcount_end(&devnet_rename_seq);
  23641. + mutex_unlock(&devnet_rename_mutex);
  23642. netdev_adjacent_rename_links(dev, oldname);
  23643. @@ -1206,7 +1206,8 @@
  23644. /* err >= 0 after dev_alloc_name() or stores the first errno */
  23645. if (err >= 0) {
  23646. err = ret;
  23647. - write_seqcount_begin(&devnet_rename_seq);
  23648. + mutex_lock(&devnet_rename_mutex);
  23649. + __raw_write_seqcount_begin(&devnet_rename_seq);
  23650. memcpy(dev->name, oldname, IFNAMSIZ);
  23651. memcpy(oldname, newname, IFNAMSIZ);
  23652. dev->name_assign_type = old_assign_type;
  23653. @@ -1219,6 +1220,11 @@
  23654. }
  23655. return err;
  23656. +
  23657. +outunlock:
  23658. + __raw_write_seqcount_end(&devnet_rename_seq);
  23659. + mutex_unlock(&devnet_rename_mutex);
  23660. + return err;
  23661. }
  23662. /**
  23663. @@ -2264,6 +2270,7 @@
  23664. sd->output_queue_tailp = &q->next_sched;
  23665. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  23666. local_irq_restore(flags);
  23667. + preempt_check_resched_rt();
  23668. }
  23669. void __netif_schedule(struct Qdisc *q)
  23670. @@ -2345,6 +2352,7 @@
  23671. __this_cpu_write(softnet_data.completion_queue, skb);
  23672. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  23673. local_irq_restore(flags);
  23674. + preempt_check_resched_rt();
  23675. }
  23676. EXPORT_SYMBOL(__dev_kfree_skb_irq);
  23677. @@ -3035,7 +3043,11 @@
  23678. * This permits __QDISC___STATE_RUNNING owner to get the lock more
  23679. * often and dequeue packets faster.
  23680. */
  23681. +#ifdef CONFIG_PREEMPT_RT_FULL
  23682. + contended = true;
  23683. +#else
  23684. contended = qdisc_is_running(q);
  23685. +#endif
  23686. if (unlikely(contended))
  23687. spin_lock(&q->busylock);
  23688. @@ -3096,9 +3108,44 @@
  23689. #define skb_update_prio(skb)
  23690. #endif
  23691. +#ifdef CONFIG_PREEMPT_RT_FULL
  23692. +
  23693. +static inline int xmit_rec_read(void)
  23694. +{
  23695. + return current->xmit_recursion;
  23696. +}
  23697. +
  23698. +static inline void xmit_rec_inc(void)
  23699. +{
  23700. + current->xmit_recursion++;
  23701. +}
  23702. +
  23703. +static inline void xmit_rec_dec(void)
  23704. +{
  23705. + current->xmit_recursion--;
  23706. +}
  23707. +
  23708. +#else
  23709. +
  23710. DEFINE_PER_CPU(int, xmit_recursion);
  23711. EXPORT_SYMBOL(xmit_recursion);
  23712. +static inline int xmit_rec_read(void)
  23713. +{
  23714. + return __this_cpu_read(xmit_recursion);
  23715. +}
  23716. +
  23717. +static inline void xmit_rec_inc(void)
  23718. +{
  23719. + __this_cpu_inc(xmit_recursion);
  23720. +}
  23721. +
  23722. +static inline void xmit_rec_dec(void)
  23723. +{
  23724. + __this_cpu_dec(xmit_recursion);
  23725. +}
  23726. +#endif
  23727. +
  23728. #define RECURSION_LIMIT 10
  23729. /**
  23730. @@ -3344,7 +3391,7 @@
  23731. if (txq->xmit_lock_owner != cpu) {
  23732. - if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
  23733. + if (xmit_rec_read() > RECURSION_LIMIT)
  23734. goto recursion_alert;
  23735. skb = validate_xmit_skb(skb, dev);
  23736. @@ -3354,9 +3401,9 @@
  23737. HARD_TX_LOCK(dev, txq, cpu);
  23738. if (!netif_xmit_stopped(txq)) {
  23739. - __this_cpu_inc(xmit_recursion);
  23740. + xmit_rec_inc();
  23741. skb = dev_hard_start_xmit(skb, dev, txq, &rc);
  23742. - __this_cpu_dec(xmit_recursion);
  23743. + xmit_rec_dec();
  23744. if (dev_xmit_complete(rc)) {
  23745. HARD_TX_UNLOCK(dev, txq);
  23746. goto out;
  23747. @@ -3730,6 +3777,7 @@
  23748. rps_unlock(sd);
  23749. local_irq_restore(flags);
  23750. + preempt_check_resched_rt();
  23751. atomic_long_inc(&skb->dev->rx_dropped);
  23752. kfree_skb(skb);
  23753. @@ -3748,7 +3796,7 @@
  23754. struct rps_dev_flow voidflow, *rflow = &voidflow;
  23755. int cpu;
  23756. - preempt_disable();
  23757. + migrate_disable();
  23758. rcu_read_lock();
  23759. cpu = get_rps_cpu(skb->dev, skb, &rflow);
  23760. @@ -3758,13 +3806,13 @@
  23761. ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
  23762. rcu_read_unlock();
  23763. - preempt_enable();
  23764. + migrate_enable();
  23765. } else
  23766. #endif
  23767. {
  23768. unsigned int qtail;
  23769. - ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
  23770. - put_cpu();
  23771. + ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail);
  23772. + put_cpu_light();
  23773. }
  23774. return ret;
  23775. }
  23776. @@ -3798,16 +3846,44 @@
  23777. trace_netif_rx_ni_entry(skb);
  23778. - preempt_disable();
  23779. + local_bh_disable();
  23780. err = netif_rx_internal(skb);
  23781. - if (local_softirq_pending())
  23782. - do_softirq();
  23783. - preempt_enable();
  23784. + local_bh_enable();
  23785. return err;
  23786. }
  23787. EXPORT_SYMBOL(netif_rx_ni);
  23788. +#ifdef CONFIG_PREEMPT_RT_FULL
  23789. +/*
  23790. + * RT runs ksoftirqd as a real time thread and the root_lock is a
  23791. + * "sleeping spinlock". If the trylock fails then we can go into an
  23792. + * infinite loop when ksoftirqd preempted the task which actually
  23793. + * holds the lock, because we requeue q and raise NET_TX softirq
  23794. + * causing ksoftirqd to loop forever.
  23795. + *
  23796. + * It's safe to use spin_lock on RT here as softirqs run in thread
  23797. + * context and cannot deadlock against the thread which is holding
  23798. + * root_lock.
  23799. + *
  23800. + * On !RT the trylock might fail, but there we bail out from the
  23801. + * softirq loop after 10 attempts which we can't do on RT. And the
  23802. + * task holding root_lock cannot be preempted, so the only downside of
  23803. + * that trylock is that we need 10 loops to decide that we should have
  23804. + * given up in the first one :)
  23805. + */
  23806. +static inline int take_root_lock(spinlock_t *lock)
  23807. +{
  23808. + spin_lock(lock);
  23809. + return 1;
  23810. +}
  23811. +#else
  23812. +static inline int take_root_lock(spinlock_t *lock)
  23813. +{
  23814. + return spin_trylock(lock);
  23815. +}
  23816. +#endif
  23817. +
  23818. static void net_tx_action(struct softirq_action *h)
  23819. {
  23820. struct softnet_data *sd = this_cpu_ptr(&softnet_data);
  23821. @@ -3855,7 +3931,7 @@
  23822. head = head->next_sched;
  23823. root_lock = qdisc_lock(q);
  23824. - if (spin_trylock(root_lock)) {
  23825. + if (take_root_lock(root_lock)) {
  23826. smp_mb__before_atomic();
  23827. clear_bit(__QDISC_STATE_SCHED,
  23828. &q->state);
  23829. @@ -4264,7 +4340,7 @@
  23830. skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
  23831. if (skb->dev == dev) {
  23832. __skb_unlink(skb, &sd->input_pkt_queue);
  23833. - kfree_skb(skb);
  23834. + __skb_queue_tail(&sd->tofree_queue, skb);
  23835. input_queue_head_incr(sd);
  23836. }
  23837. }
  23838. @@ -4273,10 +4349,13 @@
  23839. skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
  23840. if (skb->dev == dev) {
  23841. __skb_unlink(skb, &sd->process_queue);
  23842. - kfree_skb(skb);
  23843. + __skb_queue_tail(&sd->tofree_queue, skb);
  23844. input_queue_head_incr(sd);
  23845. }
  23846. }
  23847. +
  23848. + if (!skb_queue_empty(&sd->tofree_queue))
  23849. + raise_softirq_irqoff(NET_RX_SOFTIRQ);
  23850. }
  23851. static int napi_gro_complete(struct sk_buff *skb)
  23852. @@ -4735,6 +4814,7 @@
  23853. sd->rps_ipi_list = NULL;
  23854. local_irq_enable();
  23855. + preempt_check_resched_rt();
  23856. /* Send pending IPI's to kick RPS processing on remote cpus. */
  23857. while (remsd) {
  23858. @@ -4748,6 +4828,7 @@
  23859. } else
  23860. #endif
  23861. local_irq_enable();
  23862. + preempt_check_resched_rt();
  23863. }
  23864. static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
  23865. @@ -4829,6 +4910,7 @@
  23866. local_irq_save(flags);
  23867. ____napi_schedule(this_cpu_ptr(&softnet_data), n);
  23868. local_irq_restore(flags);
  23869. + preempt_check_resched_rt();
  23870. }
  23871. EXPORT_SYMBOL(__napi_schedule);
  23872. @@ -5169,7 +5251,7 @@
  23873. list_splice_tail(&repoll, &list);
  23874. list_splice(&list, &sd->poll_list);
  23875. if (!list_empty(&sd->poll_list))
  23876. - __raise_softirq_irqoff(NET_RX_SOFTIRQ);
  23877. + __raise_softirq_irqoff_ksoft(NET_RX_SOFTIRQ);
  23878. net_rps_action_and_irq_enable(sd);
  23879. }
  23880. @@ -7534,7 +7616,7 @@
  23881. void synchronize_net(void)
  23882. {
  23883. might_sleep();
  23884. - if (rtnl_is_locked())
  23885. + if (rtnl_is_locked() && !IS_ENABLED(CONFIG_PREEMPT_RT_FULL))
  23886. synchronize_rcu_expedited();
  23887. else
  23888. synchronize_rcu();
  23889. @@ -7775,16 +7857,20 @@
  23890. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  23891. local_irq_enable();
  23892. + preempt_check_resched_rt();
  23893. /* Process offline CPU's input_pkt_queue */
  23894. while ((skb = __skb_dequeue(&oldsd->process_queue))) {
  23895. netif_rx_ni(skb);
  23896. input_queue_head_incr(oldsd);
  23897. }
  23898. - while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
  23899. + while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
  23900. netif_rx_ni(skb);
  23901. input_queue_head_incr(oldsd);
  23902. }
  23903. + while ((skb = __skb_dequeue(&oldsd->tofree_queue))) {
  23904. + kfree_skb(skb);
  23905. + }
  23906. return NOTIFY_OK;
  23907. }
  23908. @@ -8086,8 +8172,9 @@
  23909. for_each_possible_cpu(i) {
  23910. struct softnet_data *sd = &per_cpu(softnet_data, i);
  23911. - skb_queue_head_init(&sd->input_pkt_queue);
  23912. - skb_queue_head_init(&sd->process_queue);
  23913. + skb_queue_head_init_raw(&sd->input_pkt_queue);
  23914. + skb_queue_head_init_raw(&sd->process_queue);
  23915. + skb_queue_head_init_raw(&sd->tofree_queue);
  23916. INIT_LIST_HEAD(&sd->poll_list);
  23917. sd->output_queue_tailp = &sd->output_queue;
  23918. #ifdef CONFIG_RPS
  23919. diff -Nur linux-4.6.6.orig/net/core/skbuff.c linux-4.6.6/net/core/skbuff.c
  23920. --- linux-4.6.6.orig/net/core/skbuff.c 2016-08-10 12:54:57.000000000 +0200
  23921. +++ linux-4.6.6/net/core/skbuff.c 2016-08-22 21:27:47.449895420 +0200
  23922. @@ -63,6 +63,7 @@
  23923. #include <linux/errqueue.h>
  23924. #include <linux/prefetch.h>
  23925. #include <linux/if_vlan.h>
  23926. +#include <linux/locallock.h>
  23927. #include <net/protocol.h>
  23928. #include <net/dst.h>
  23929. @@ -359,6 +360,8 @@
  23930. static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
  23931. static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
  23932. +static DEFINE_LOCAL_IRQ_LOCK(netdev_alloc_lock);
  23933. +static DEFINE_LOCAL_IRQ_LOCK(napi_alloc_cache_lock);
  23934. static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
  23935. {
  23936. @@ -366,10 +369,10 @@
  23937. unsigned long flags;
  23938. void *data;
  23939. - local_irq_save(flags);
  23940. + local_lock_irqsave(netdev_alloc_lock, flags);
  23941. nc = this_cpu_ptr(&netdev_alloc_cache);
  23942. data = __alloc_page_frag(nc, fragsz, gfp_mask);
  23943. - local_irq_restore(flags);
  23944. + local_unlock_irqrestore(netdev_alloc_lock, flags);
  23945. return data;
  23946. }
  23947. @@ -388,9 +391,13 @@
  23948. static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
  23949. {
  23950. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  23951. + struct napi_alloc_cache *nc;
  23952. + void *data;
  23953. - return __alloc_page_frag(&nc->page, fragsz, gfp_mask);
  23954. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23955. + data = __alloc_page_frag(&nc->page, fragsz, gfp_mask);
  23956. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23957. + return data;
  23958. }
  23959. void *napi_alloc_frag(unsigned int fragsz)
  23960. @@ -437,13 +444,13 @@
  23961. if (sk_memalloc_socks())
  23962. gfp_mask |= __GFP_MEMALLOC;
  23963. - local_irq_save(flags);
  23964. + local_lock_irqsave(netdev_alloc_lock, flags);
  23965. nc = this_cpu_ptr(&netdev_alloc_cache);
  23966. data = __alloc_page_frag(nc, len, gfp_mask);
  23967. pfmemalloc = nc->pfmemalloc;
  23968. - local_irq_restore(flags);
  23969. + local_unlock_irqrestore(netdev_alloc_lock, flags);
  23970. if (unlikely(!data))
  23971. return NULL;
  23972. @@ -484,9 +491,10 @@
  23973. struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
  23974. gfp_t gfp_mask)
  23975. {
  23976. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  23977. + struct napi_alloc_cache *nc;
  23978. struct sk_buff *skb;
  23979. void *data;
  23980. + bool pfmemalloc;
  23981. len += NET_SKB_PAD + NET_IP_ALIGN;
  23982. @@ -504,7 +512,10 @@
  23983. if (sk_memalloc_socks())
  23984. gfp_mask |= __GFP_MEMALLOC;
  23985. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23986. data = __alloc_page_frag(&nc->page, len, gfp_mask);
  23987. + pfmemalloc = nc->page.pfmemalloc;
  23988. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23989. if (unlikely(!data))
  23990. return NULL;
  23991. @@ -515,7 +526,7 @@
  23992. }
  23993. /* use OR instead of assignment to avoid clearing of bits in mask */
  23994. - if (nc->page.pfmemalloc)
  23995. + if (pfmemalloc)
  23996. skb->pfmemalloc = 1;
  23997. skb->head_frag = 1;
  23998. @@ -759,23 +770,26 @@
  23999. void __kfree_skb_flush(void)
  24000. {
  24001. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  24002. + struct napi_alloc_cache *nc;
  24003. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  24004. /* flush skb_cache if containing objects */
  24005. if (nc->skb_count) {
  24006. kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
  24007. nc->skb_cache);
  24008. nc->skb_count = 0;
  24009. }
  24010. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  24011. }
  24012. static inline void _kfree_skb_defer(struct sk_buff *skb)
  24013. {
  24014. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  24015. + struct napi_alloc_cache *nc;
  24016. /* drop skb->head and call any destructors for packet */
  24017. skb_release_all(skb);
  24018. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  24019. /* record skb to CPU local list */
  24020. nc->skb_cache[nc->skb_count++] = skb;
  24021. @@ -790,6 +804,7 @@
  24022. nc->skb_cache);
  24023. nc->skb_count = 0;
  24024. }
  24025. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  24026. }
  24027. void __kfree_skb_defer(struct sk_buff *skb)
  24028. {
  24029. diff -Nur linux-4.6.6.orig/net/core/sock.c linux-4.6.6/net/core/sock.c
  24030. --- linux-4.6.6.orig/net/core/sock.c 2016-08-10 12:54:57.000000000 +0200
  24031. +++ linux-4.6.6/net/core/sock.c 2016-08-22 21:27:47.453895680 +0200
  24032. @@ -2421,12 +2421,11 @@
  24033. if (sk->sk_lock.owned)
  24034. __lock_sock(sk);
  24035. sk->sk_lock.owned = 1;
  24036. - spin_unlock(&sk->sk_lock.slock);
  24037. + spin_unlock_bh(&sk->sk_lock.slock);
  24038. /*
  24039. * The sk_lock has mutex_lock() semantics here:
  24040. */
  24041. mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
  24042. - local_bh_enable();
  24043. }
  24044. EXPORT_SYMBOL(lock_sock_nested);
  24045. diff -Nur linux-4.6.6.orig/net/ipv4/icmp.c linux-4.6.6/net/ipv4/icmp.c
  24046. --- linux-4.6.6.orig/net/ipv4/icmp.c 2016-08-10 12:54:57.000000000 +0200
  24047. +++ linux-4.6.6/net/ipv4/icmp.c 2016-08-22 21:27:47.453895680 +0200
  24048. @@ -69,6 +69,7 @@
  24049. #include <linux/jiffies.h>
  24050. #include <linux/kernel.h>
  24051. #include <linux/fcntl.h>
  24052. +#include <linux/sysrq.h>
  24053. #include <linux/socket.h>
  24054. #include <linux/in.h>
  24055. #include <linux/inet.h>
  24056. @@ -891,6 +892,30 @@
  24057. }
  24058. /*
  24059. + * 32bit and 64bit have different timestamp length, so we check for
  24060. + * the cookie at offset 20 and verify it is repeated at offset 50
  24061. + */
  24062. +#define CO_POS0 20
  24063. +#define CO_POS1 50
  24064. +#define CO_SIZE sizeof(int)
  24065. +#define ICMP_SYSRQ_SIZE 57
  24066. +
  24067. +/*
  24068. + * We got a ICMP_SYSRQ_SIZE sized ping request. Check for the cookie
  24069. + * pattern and if it matches send the next byte as a trigger to sysrq.
  24070. + */
  24071. +static void icmp_check_sysrq(struct net *net, struct sk_buff *skb)
  24072. +{
  24073. + int cookie = htonl(net->ipv4.sysctl_icmp_echo_sysrq);
  24074. + char *p = skb->data;
  24075. +
  24076. + if (!memcmp(&cookie, p + CO_POS0, CO_SIZE) &&
  24077. + !memcmp(&cookie, p + CO_POS1, CO_SIZE) &&
  24078. + p[CO_POS0 + CO_SIZE] == p[CO_POS1 + CO_SIZE])
  24079. + handle_sysrq(p[CO_POS0 + CO_SIZE]);
  24080. +}
  24081. +
  24082. +/*
  24083. * Handle ICMP_ECHO ("ping") requests.
  24084. *
  24085. * RFC 1122: 3.2.2.6 MUST have an echo server that answers ICMP echo
  24086. @@ -917,6 +942,11 @@
  24087. icmp_param.data_len = skb->len;
  24088. icmp_param.head_len = sizeof(struct icmphdr);
  24089. icmp_reply(&icmp_param, skb);
  24090. +
  24091. + if (skb->len == ICMP_SYSRQ_SIZE &&
  24092. + net->ipv4.sysctl_icmp_echo_sysrq) {
  24093. + icmp_check_sysrq(net, skb);
  24094. + }
  24095. }
  24096. /* should there be an ICMP stat for ignored echos? */
  24097. return true;
  24098. diff -Nur linux-4.6.6.orig/net/ipv4/inet_connection_sock.c linux-4.6.6/net/ipv4/inet_connection_sock.c
  24099. --- linux-4.6.6.orig/net/ipv4/inet_connection_sock.c 2016-08-10 12:54:57.000000000 +0200
  24100. +++ linux-4.6.6/net/ipv4/inet_connection_sock.c 2016-08-22 21:27:47.453895680 +0200
  24101. @@ -603,7 +603,7 @@
  24102. if (req->num_timeout++ == 0)
  24103. atomic_dec(&queue->young);
  24104. timeo = min(TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX);
  24105. - mod_timer_pinned(&req->rsk_timer, jiffies + timeo);
  24106. + mod_timer(&req->rsk_timer, jiffies + timeo);
  24107. return;
  24108. }
  24109. drop:
  24110. @@ -617,8 +617,9 @@
  24111. req->num_timeout = 0;
  24112. req->sk = NULL;
  24113. - setup_timer(&req->rsk_timer, reqsk_timer_handler, (unsigned long)req);
  24114. - mod_timer_pinned(&req->rsk_timer, jiffies + timeout);
  24115. + setup_pinned_timer(&req->rsk_timer, reqsk_timer_handler,
  24116. + (unsigned long)req);
  24117. + mod_timer(&req->rsk_timer, jiffies + timeout);
  24118. inet_ehash_insert(req_to_sk(req), NULL);
  24119. /* before letting lookups find us, make sure all req fields
  24120. diff -Nur linux-4.6.6.orig/net/ipv4/inet_timewait_sock.c linux-4.6.6/net/ipv4/inet_timewait_sock.c
  24121. --- linux-4.6.6.orig/net/ipv4/inet_timewait_sock.c 2016-08-10 12:54:57.000000000 +0200
  24122. +++ linux-4.6.6/net/ipv4/inet_timewait_sock.c 2016-08-22 21:27:47.453895680 +0200
  24123. @@ -188,7 +188,8 @@
  24124. tw->tw_prot = sk->sk_prot_creator;
  24125. atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
  24126. twsk_net_set(tw, sock_net(sk));
  24127. - setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
  24128. + setup_pinned_timer(&tw->tw_timer, tw_timer_handler,
  24129. + (unsigned long)tw);
  24130. /*
  24131. * Because we use RCU lookups, we should not set tw_refcnt
  24132. * to a non null value before everything is setup for this
  24133. @@ -248,7 +249,7 @@
  24134. tw->tw_kill = timeo <= 4*HZ;
  24135. if (!rearm) {
  24136. - BUG_ON(mod_timer_pinned(&tw->tw_timer, jiffies + timeo));
  24137. + BUG_ON(mod_timer(&tw->tw_timer, jiffies + timeo));
  24138. atomic_inc(&tw->tw_dr->tw_count);
  24139. } else {
  24140. mod_timer_pending(&tw->tw_timer, jiffies + timeo);
  24141. diff -Nur linux-4.6.6.orig/net/ipv4/sysctl_net_ipv4.c linux-4.6.6/net/ipv4/sysctl_net_ipv4.c
  24142. --- linux-4.6.6.orig/net/ipv4/sysctl_net_ipv4.c 2016-08-10 12:54:57.000000000 +0200
  24143. +++ linux-4.6.6/net/ipv4/sysctl_net_ipv4.c 2016-08-22 21:27:47.453895680 +0200
  24144. @@ -681,6 +681,13 @@
  24145. .proc_handler = proc_dointvec
  24146. },
  24147. {
  24148. + .procname = "icmp_echo_sysrq",
  24149. + .data = &init_net.ipv4.sysctl_icmp_echo_sysrq,
  24150. + .maxlen = sizeof(int),
  24151. + .mode = 0644,
  24152. + .proc_handler = proc_dointvec
  24153. + },
  24154. + {
  24155. .procname = "icmp_ignore_bogus_error_responses",
  24156. .data = &init_net.ipv4.sysctl_icmp_ignore_bogus_error_responses,
  24157. .maxlen = sizeof(int),
  24158. diff -Nur linux-4.6.6.orig/net/mac80211/rx.c linux-4.6.6/net/mac80211/rx.c
  24159. --- linux-4.6.6.orig/net/mac80211/rx.c 2016-08-10 12:54:57.000000000 +0200
  24160. +++ linux-4.6.6/net/mac80211/rx.c 2016-08-22 21:27:47.453895680 +0200
  24161. @@ -3679,7 +3679,7 @@
  24162. struct ieee80211_supported_band *sband;
  24163. struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
  24164. - WARN_ON_ONCE(softirq_count() == 0);
  24165. + WARN_ON_ONCE_NONRT(softirq_count() == 0);
  24166. if (WARN_ON(status->band >= IEEE80211_NUM_BANDS))
  24167. goto drop;
  24168. diff -Nur linux-4.6.6.orig/net/netfilter/core.c linux-4.6.6/net/netfilter/core.c
  24169. --- linux-4.6.6.orig/net/netfilter/core.c 2016-08-10 12:54:57.000000000 +0200
  24170. +++ linux-4.6.6/net/netfilter/core.c 2016-08-22 21:27:47.453895680 +0200
  24171. @@ -22,11 +22,17 @@
  24172. #include <linux/proc_fs.h>
  24173. #include <linux/mutex.h>
  24174. #include <linux/slab.h>
  24175. +#include <linux/locallock.h>
  24176. #include <net/net_namespace.h>
  24177. #include <net/sock.h>
  24178. #include "nf_internals.h"
  24179. +#ifdef CONFIG_PREEMPT_RT_BASE
  24180. +DEFINE_LOCAL_IRQ_LOCK(xt_write_lock);
  24181. +EXPORT_PER_CPU_SYMBOL(xt_write_lock);
  24182. +#endif
  24183. +
  24184. static DEFINE_MUTEX(afinfo_mutex);
  24185. const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly;
  24186. diff -Nur linux-4.6.6.orig/net/packet/af_packet.c linux-4.6.6/net/packet/af_packet.c
  24187. --- linux-4.6.6.orig/net/packet/af_packet.c 2016-08-10 12:54:57.000000000 +0200
  24188. +++ linux-4.6.6/net/packet/af_packet.c 2016-08-22 21:27:47.457895933 +0200
  24189. @@ -63,6 +63,7 @@
  24190. #include <linux/if_packet.h>
  24191. #include <linux/wireless.h>
  24192. #include <linux/kernel.h>
  24193. +#include <linux/delay.h>
  24194. #include <linux/kmod.h>
  24195. #include <linux/slab.h>
  24196. #include <linux/vmalloc.h>
  24197. @@ -694,7 +695,7 @@
  24198. if (BLOCK_NUM_PKTS(pbd)) {
  24199. while (atomic_read(&pkc->blk_fill_in_prog)) {
  24200. /* Waiting for skb_copy_bits to finish... */
  24201. - cpu_relax();
  24202. + cpu_chill();
  24203. }
  24204. }
  24205. @@ -956,7 +957,7 @@
  24206. if (!(status & TP_STATUS_BLK_TMO)) {
  24207. while (atomic_read(&pkc->blk_fill_in_prog)) {
  24208. /* Waiting for skb_copy_bits to finish... */
  24209. - cpu_relax();
  24210. + cpu_chill();
  24211. }
  24212. }
  24213. prb_close_block(pkc, pbd, po, status);
  24214. diff -Nur linux-4.6.6.orig/net/rds/ib_rdma.c linux-4.6.6/net/rds/ib_rdma.c
  24215. --- linux-4.6.6.orig/net/rds/ib_rdma.c 2016-08-10 12:54:57.000000000 +0200
  24216. +++ linux-4.6.6/net/rds/ib_rdma.c 2016-08-22 21:27:47.457895933 +0200
  24217. @@ -34,6 +34,7 @@
  24218. #include <linux/slab.h>
  24219. #include <linux/rculist.h>
  24220. #include <linux/llist.h>
  24221. +#include <linux/delay.h>
  24222. #include "ib_mr.h"
  24223. @@ -209,7 +210,7 @@
  24224. for_each_online_cpu(cpu) {
  24225. flag = &per_cpu(clean_list_grace, cpu);
  24226. while (test_bit(CLEAN_LIST_BUSY_BIT, flag))
  24227. - cpu_relax();
  24228. + cpu_chill();
  24229. }
  24230. }
  24231. diff -Nur linux-4.6.6.orig/net/sched/sch_generic.c linux-4.6.6/net/sched/sch_generic.c
  24232. --- linux-4.6.6.orig/net/sched/sch_generic.c 2016-08-10 12:54:57.000000000 +0200
  24233. +++ linux-4.6.6/net/sched/sch_generic.c 2016-08-22 21:27:47.457895933 +0200
  24234. @@ -894,7 +894,7 @@
  24235. /* Wait for outstanding qdisc_run calls. */
  24236. list_for_each_entry(dev, head, close_list)
  24237. while (some_qdisc_is_busy(dev))
  24238. - yield();
  24239. + msleep(1);
  24240. }
  24241. void dev_deactivate(struct net_device *dev)
  24242. diff -Nur linux-4.6.6.orig/net/sunrpc/svc_xprt.c linux-4.6.6/net/sunrpc/svc_xprt.c
  24243. --- linux-4.6.6.orig/net/sunrpc/svc_xprt.c 2016-08-10 12:54:57.000000000 +0200
  24244. +++ linux-4.6.6/net/sunrpc/svc_xprt.c 2016-08-22 21:27:47.457895933 +0200
  24245. @@ -344,7 +344,7 @@
  24246. goto out;
  24247. }
  24248. - cpu = get_cpu();
  24249. + cpu = get_cpu_light();
  24250. pool = svc_pool_for_cpu(xprt->xpt_server, cpu);
  24251. atomic_long_inc(&pool->sp_stats.packets);
  24252. @@ -380,7 +380,7 @@
  24253. atomic_long_inc(&pool->sp_stats.threads_woken);
  24254. wake_up_process(rqstp->rq_task);
  24255. - put_cpu();
  24256. + put_cpu_light();
  24257. goto out;
  24258. }
  24259. rcu_read_unlock();
  24260. @@ -401,7 +401,7 @@
  24261. goto redo_search;
  24262. }
  24263. rqstp = NULL;
  24264. - put_cpu();
  24265. + put_cpu_light();
  24266. out:
  24267. trace_svc_xprt_do_enqueue(xprt, rqstp);
  24268. }
  24269. diff -Nur linux-4.6.6.orig/scripts/mkcompile_h linux-4.6.6/scripts/mkcompile_h
  24270. --- linux-4.6.6.orig/scripts/mkcompile_h 2016-08-10 12:54:57.000000000 +0200
  24271. +++ linux-4.6.6/scripts/mkcompile_h 2016-08-22 21:27:47.457895933 +0200
  24272. @@ -4,7 +4,8 @@
  24273. ARCH=$2
  24274. SMP=$3
  24275. PREEMPT=$4
  24276. -CC=$5
  24277. +RT=$5
  24278. +CC=$6
  24279. vecho() { [ "${quiet}" = "silent_" ] || echo "$@" ; }
  24280. @@ -57,6 +58,7 @@
  24281. CONFIG_FLAGS=""
  24282. if [ -n "$SMP" ] ; then CONFIG_FLAGS="SMP"; fi
  24283. if [ -n "$PREEMPT" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT"; fi
  24284. +if [ -n "$RT" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS RT"; fi
  24285. UTS_VERSION="$UTS_VERSION $CONFIG_FLAGS $TIMESTAMP"
  24286. # Truncate to maximum length
  24287. diff -Nur linux-4.6.6.orig/sound/core/pcm_native.c linux-4.6.6/sound/core/pcm_native.c
  24288. --- linux-4.6.6.orig/sound/core/pcm_native.c 2016-08-10 12:54:57.000000000 +0200
  24289. +++ linux-4.6.6/sound/core/pcm_native.c 2016-08-22 21:27:47.457895933 +0200
  24290. @@ -135,7 +135,7 @@
  24291. void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream)
  24292. {
  24293. if (!substream->pcm->nonatomic)
  24294. - local_irq_disable();
  24295. + local_irq_disable_nort();
  24296. snd_pcm_stream_lock(substream);
  24297. }
  24298. EXPORT_SYMBOL_GPL(snd_pcm_stream_lock_irq);
  24299. @@ -150,7 +150,7 @@
  24300. {
  24301. snd_pcm_stream_unlock(substream);
  24302. if (!substream->pcm->nonatomic)
  24303. - local_irq_enable();
  24304. + local_irq_enable_nort();
  24305. }
  24306. EXPORT_SYMBOL_GPL(snd_pcm_stream_unlock_irq);
  24307. @@ -158,7 +158,7 @@
  24308. {
  24309. unsigned long flags = 0;
  24310. if (!substream->pcm->nonatomic)
  24311. - local_irq_save(flags);
  24312. + local_irq_save_nort(flags);
  24313. snd_pcm_stream_lock(substream);
  24314. return flags;
  24315. }
  24316. @@ -176,7 +176,7 @@
  24317. {
  24318. snd_pcm_stream_unlock(substream);
  24319. if (!substream->pcm->nonatomic)
  24320. - local_irq_restore(flags);
  24321. + local_irq_restore_nort(flags);
  24322. }
  24323. EXPORT_SYMBOL_GPL(snd_pcm_stream_unlock_irqrestore);